From 5a1062b30450b879ad29419b63adb6634d97e183 Mon Sep 17 00:00:00 2001 From: Stijn Moreels <9039753+stijnmoreels@users.noreply.github.com> Date: Mon, 9 Jan 2023 13:08:23 +0100 Subject: [PATCH] docs: publish v2.7 feature docs (#512) --- .../versioned_docs/version-v2.7.0/01-index.md | 32 + .../Service-to-service Correlation/index.md | 42 ++ ...e-to-service-correlation-in-service-bus.md | 355 +++++++++ ...rvice-to-service-correlation-in-web-api.md | 329 +++++++++ .../use-with-dotnet-and-aspnetcore.md | 158 ++++ .../use-with-dotnet-and-functions.md | 123 ++++ .../version-v2.7.0/03-Features/correlation.md | 109 +++ .../making-telemetry-more-powerful.md | 64 ++ .../sinks/azure-application-insights.md | 230 ++++++ .../03-Features/telemetry-enrichment.md | 344 +++++++++ .../03-Features/telemetry-filter.md | 54 ++ .../writing-different-telemetry-types.md | 697 ++++++++++++++++++ .../version-v2.7.0-sidebars.json | 8 + docs/versions.json | 1 + 14 files changed, 2546 insertions(+) create mode 100644 docs/versioned_docs/version-v2.7.0/01-index.md create mode 100644 docs/versioned_docs/version-v2.7.0/02-Guidance/Service-to-service Correlation/index.md create mode 100644 docs/versioned_docs/version-v2.7.0/02-Guidance/Service-to-service Correlation/use-service-to-service-correlation-in-service-bus.md create mode 100644 docs/versioned_docs/version-v2.7.0/02-Guidance/Service-to-service Correlation/use-service-to-service-correlation-in-web-api.md create mode 100644 docs/versioned_docs/version-v2.7.0/02-Guidance/use-with-dotnet-and-aspnetcore.md create mode 100644 docs/versioned_docs/version-v2.7.0/02-Guidance/use-with-dotnet-and-functions.md create mode 100644 docs/versioned_docs/version-v2.7.0/03-Features/correlation.md create mode 100644 docs/versioned_docs/version-v2.7.0/03-Features/making-telemetry-more-powerful.md create mode 100644 docs/versioned_docs/version-v2.7.0/03-Features/sinks/azure-application-insights.md create mode 100644 docs/versioned_docs/version-v2.7.0/03-Features/telemetry-enrichment.md create mode 100644 docs/versioned_docs/version-v2.7.0/03-Features/telemetry-filter.md create mode 100644 docs/versioned_docs/version-v2.7.0/03-Features/writing-different-telemetry-types.md create mode 100644 docs/versioned_sidebars/version-v2.7.0-sidebars.json diff --git a/docs/versioned_docs/version-v2.7.0/01-index.md b/docs/versioned_docs/version-v2.7.0/01-index.md new file mode 100644 index 00000000..5c02b855 --- /dev/null +++ b/docs/versioned_docs/version-v2.7.0/01-index.md @@ -0,0 +1,32 @@ +--- +title: "Arcus Observability" +layout: default +slug: / +sidebar_label: Welcome +--- + +# Introduction + +Arcus Observability allows you to work easily with Azure Application Insights telemetry by making use of the common `ILogger` infrastructure to track dependencies, log custom metrics and log multi-dimensional telemetry data. The library supports multiple telemetry types like tracking dependencies, requests, events, metrics, while also be able to filter with Serilog filters and enrich with custom correlation. + +![Logger Arcus - Application Insights](/img/logger-arcus-appinsights.png) + +# Guidance + +- [Using Arcus & Serilog in ASP.NET Core](./02-Guidance/use-with-dotnet-and-aspnetcore.md) +- [Using Arcus & Serilog in Azure Functions](./02-Guidance/use-with-dotnet-and-functions.md) + +# Installation + +The Arcus.Observability.Correlation package can be installed via NuGet: + +```shell +PM > Install-Package Arcus.Observability.Serilog.Sinks.ApplicationInsights +``` + +For more granular packages we recommend reading [the documentation](./03-Features/sinks/azure-application-insights.md). + +# License +This is licensed under The MIT License (MIT). Which means that you can use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the web application. But you always need to state that Codit is the original author of this web application. + +*[Full license here](https://github.com/arcus-azure/arcus.observability/blob/master/LICENSE)* diff --git a/docs/versioned_docs/version-v2.7.0/02-Guidance/Service-to-service Correlation/index.md b/docs/versioned_docs/version-v2.7.0/02-Guidance/Service-to-service Correlation/index.md new file mode 100644 index 00000000..16d98f5e --- /dev/null +++ b/docs/versioned_docs/version-v2.7.0/02-Guidance/Service-to-service Correlation/index.md @@ -0,0 +1,42 @@ +--- +title: "Service-to-service correlation?" +layout: default +--- + +# Service-to-service correlation +The concept of service-to-service correlation is big and complex and spans multiple Arcus libraries. This user-guide will walk through all the available Arcus features that work together to facilitate service-to-service correlation in your application. + +## What is service-to-service correlation? +Service-to-service correlation is a way to describe a relationship between different components where each 'component' represents a separate application. This could be, for example, an API call that results in an Azure Service Bus message that will be picked up by a message pump, or an Azure Function that calls an additional API after receiving a message from a queue. + +Anytime one application calls another, that's where service-to-service correlation comes in. This way of correlation makes sure that the relationship of a single (business) transaction doesn't stop at the borders of one application or component, but continues in the other application. The end result is a clear overview of all the components involved. + +![Arcus service-to-service application map](/media/service-to-service-correlation-application-map.png) + +When using Arcus for service-to-service correlation, you'll see the this relationship in Application Insights in the [end-to-end transaction view](https://docs.microsoft.com/en-us/azure/azure-monitor/app/transaction-diagnostics): + +![Arcus service-to-service correlation relationship](/media/service-to-service-correlation-relationship.png) + +## Why should I use service-to-service correlation? +Adding service-to-service correlation to your application adds many advantages: +* ✔ Quickly spot performance bottlenecks and failures +* ✔ Clear overview of used (Azure) resources +* ✔ Effective architectural decision-making on application flow + +## What Arcus components support service-to-service correlation? +Currently, we support service-to-service correlation with several types of components: +* [Arcus Web API](https://webapi.arcus-azure.net/features/correlation) +* [Arcus Azure Service Bus message pump/router](https://messaging.arcus-azure.net/) +* And [other built-in and custom dependencies](https://observability.arcus-azure.net/Features/writing-different-telemetry-types) + +We're working on adding Azure EventHubs message pump to the mix. + +> ⚠ Service-to-service correlation is only available for Azure Functions that run in-process. Out-of-process / isolated Azure Functions doesn't have (yet) the necessary built-in Arcus functionality to facilitate service-to-service correlation. + +> ⚠ Service-to-service correlation is currently only supported for projects that uses Application Insights as their Serilog logging system. If you want to use Arcus' service-to-service functionality outside Application Insights, [please let us know](https://github.com/arcus-azure/arcus.observability/issues/new/choose). + +## How do I add service-to-service correlation to my application components? +Service-to-service correlation should be added on both the sending and receiving components of your project so the internal Arcus functionality can link the request/response correctly in Application Insights. The following user-guides will go over the sending and receiving side of Web API's and Azure Service Bus resources: + +* [Use service-to-service correlation in Web API solutions](use-service-to-service-correlation-in-web-api.md) +* [Use service-to-service correlation in Service Bus solutions](use-service-to-service-correlation-in-service-bus.md) \ No newline at end of file diff --git a/docs/versioned_docs/version-v2.7.0/02-Guidance/Service-to-service Correlation/use-service-to-service-correlation-in-service-bus.md b/docs/versioned_docs/version-v2.7.0/02-Guidance/Service-to-service Correlation/use-service-to-service-correlation-in-service-bus.md new file mode 100644 index 00000000..d7e64b03 --- /dev/null +++ b/docs/versioned_docs/version-v2.7.0/02-Guidance/Service-to-service Correlation/use-service-to-service-correlation-in-service-bus.md @@ -0,0 +1,355 @@ +--- +title: "Use service-to-service correlation in Azure Service Bus solutions" +layout: default +--- + +# Use service-to-service correlation in Azure Service Bus solutions +The concept of service-to-service correlation is big and complex and spans multiple Arcus libraries. This user-guide will walk through all the available Arcus features that work together to facilitate service-to-service correlation in your Azure Service Bus solutions. For a general introduction, see the [introduction page on service-to-service correlation](index.md). + +To use service-to-service correlation in Azure Service Bus solutions, both the sending and receiving side of the separate components have to be adapted. The correlation will be passed through from one service to another via the Azure Service Bus message application properties which Arcus will use to link your services together. + +The following diagram shows how a Web API calls a Service Bus with correlation tracking: +![Messaging correlation diagram](/media/service-to-service-api-worker-diagram-example.png) + +## How does this work? +Three kind of correlation ID's are used to create the relationship: +* **Transaction ID**: this ID is the one constant in the diagram. This ID is used to describe the entire transaction, from beginning to end. All telemetry will be tracked under this ID. +* **Operation ID**: this ID describes a single operation within the transaction. This ID is used within a service to link all telemetry of that service together. +* **Operation Parent ID**: this ID is create the parent/child link across services. When service A calls service B, then service A is the so called 'parent' of service B. + +The following list shows each step in the diagram: +1. The initial call in this example doesn't contain any correlation headers. This can be seen as a first interaction call to a service. +2. Upon receiving at service A, the application will generate new correlation information. This correlation info will be used when telemetry is tracked on the service. +3. When a call is made to service B, the **transaction ID** is sent but also the **operation parent ID** in the form of a hierarchical structure. +4. The `jkl` part of this ID, describes the new parent ID for service B (when service B calls service C, then it will use `jkl` as parent ID) +5. The user receives both the **transaction ID** and **operation parent ID** in their final response. + +> 💡 Additional configuration is available to tweak this functionality, see the [dedicated Arcus Messaging feature documentation](https://messaging.arcus-azure.net/Features/message-pumps/service-bus) for more in-depth information on Azure Service Bus correlation. + +## Service Bus demonstration +In this user-guide, a fictive API and Service Bus application will be used to represent Service A and Service B of the diagram. Both services will be adapted to use service-to-service correlation. + +* **Order API** (Service A): receives an request to order a product, sends the order to process. +* **Order Worker** (Service B): receives the order request and processes the order. + +The user interacts with the **Product API** to order their product. Internally, the **Order API** will contact the **Order Worker** to further process the order request. + +> ⚠ Take into account that this sample should only be used for demonstration purposes and does not reflect a fully production-ready implementation. We have chosen to only provide the bare bones of the application to be able to focus on the changes required for service-to-service correlation. + +### Order API: startup code +First, lets look at the initial code for the **Order API**. The startup code has the hosting and routing functionality to use API controllers. +The `Azure.Messaging.ServiceBus` package is added so that the `AddServiceBusClientWithNamespace` becomes available. This will inject a `ServiceBusClient` in the application which will contact the **Order Worker**. +```csharp +using Microsoft.Extensions.Azure; + +public class Program +{ + public static void Main(string[] args) + { + WebApplicationBuilder builder = WebApplication.CreateBuilder(args); + builder.Services.AddRouting(); + builder.Services.AddControllers(); + + builder.Services.AddAzureClients(clients => + { + clients.AddServiceBusClientWithNamespace("") + .WithName("Order Worker") + .WithCredential(new ManagedIdentityCredential()); + }); + + WebApplication app = builder.Build(); + app.UseRouting(); + app.UseEndpoints(endpoints => endpoints.MapControllers()); + app.Run("http://localhost:787"); + } +} +``` + +The sole API controller in the **Order API** makes sure that we receive the product order request and contact the **Order Worker** to further process the order request. +```csharp +using Azure.Messaging.ServiceBus; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Azure; + +[ApiController] +[Route("api/v1/order")] +public class OrderController : ControllerBase +{ + private readonly ServiceBusSender _serviceBusSender; + private readonly ILogger _logger; + + public OrderController( + IAzureClientFactory clientFactory, + ILogger logger) + { + ServiceBusClient client = clientFactory.CreateClient("Order Worker"); + _serviceBusSender = client.CreateSender("orders"); + _logger = logger; + } + + [HttpPost] + public async Task Post([FromBody] ProductOrderRequest productRequest) + { + var order = new Order(productRequest); + var message = new ServiceBusMessage(order); + + await _serviceBusSender.SendMessageAsync(message); + } +} +``` + +### Order Worker: startup code +The **Order Worker** application will use mostly Arcus functionality, so for now, the startup code is only creating the hosting functionality: +```csharp +using Microsoft.Extensions.Azure; + +public class Program +{ + public static void Main(string[] args) + { + Host.CreateDefaultBuilder(args) + .Build() + .Run(); + } +} +``` + +Right now, we haven't used any Arcus functionality, only common Microsoft-available features. + +### Order API: add Arcus correlation +Now that we have explained the application code, we can add the Arcus functionality that will provide clear telemetry for the API interaction. + +First, these packages need to be installed: +```shell +PM > Install-Package Arcus.WebApi.Logging -MinimumVersion 1.6.1 +PM > Install-Package Arcus.Observability.Telemetry.Serilog.Sinks.ApplicationInsights -MinimumVersion 2.7.0 +PM > Install-Package Azure.Messaging.ServiceBus -MinimumVersion 7.11.1 +``` + +> ⚡ Note that these Arcus packages and additions are built-in into the [Arcus project templates](https://templates.arcus-azure.net/). + +> For more information on `Arcus.WebApi.Logging`, see [these dedicated feature docs](https://webapi.arcus-azure.net/features/logging), for more information on `Arcus.Observability.Telemetry.Serilog.Sinks.ApplicationInsights`, see [these dedicated feature docs](https://observability.arcus-azure.net/Features/sinks/azure-application-insights). + +The following additions have to be made to the startup code for the **Product API** to be able to track the incoming/outgoing HTTP requests, and to log to Application Insights. +```csharp +using Microsoft.Extensions.Azure; +using Serilog; +using Serilog.Configuration; +using Serilog.Events; + +public class Program +{ + public static void Main(string[] args) + { + WebApplicationBuilder builder = WebApplication.CreateBuilder(args); + builder.Services.AddRouting(); + builder.Services.AddControllers(); + + // [Add] Makes HTTP correlation available throughout application + builder.Services.AddHttpCorrelation(); + + builder.Services.AddAzureClients(clients => + { + clients.AddServiceBusClientWithNamespace("") + .WithName("Order Worker") + .WithCredential(new ManagedIdentityCredential()); + }); + + // [Add] Arcus + Microsoft component name/version registration + builder.Services.AddAppName("Order API"); + builder.Services.AddAssemblyAppVersion(); + + // [Add] Serilog configuration that writes to Application Insights + builder.Host.UseSerilog((context, provider, config) => + { + config.MinimumLevel.Information() + .MinimumLevel.Override("Microsoft", LogEventLevel.Information) + .Enrich.WithComponentName(provider) + .Enrich.WithVersion(provider) + .Enrich.WithHttpCorrelationInfo(provider) + .WriteTo.AzureApplicationInsightsWithConnectionString(provider, ""); + }); + + WebApplication app = builder.Build(); + // [Add] Gets HTTP correlation from incoming request + app.UseHttpCorrelation(); + app.UseRouting(); + // [Add] Tracks incoming request + app.UseRequestTracking(); + app.UseEndpoints(endpoints => endpoints.MapControllers()); + app.Run("http://localhost:787"); + } +} +``` + +Following additions are made: +* `builder.Services.AddHttpCorrelation()`: adds the HTTP correlation services to the application services. This registers the `IHttpCorrelationInfoAccessor` that is used to get/set the HTTP correlation throughout the application. ([more info](https://webapi.arcus-azure.net/features/correlation)) +* `AddAppName`/`Add...AppVersion`: configures for both Arcus & Microsoft the component's name and version when tracking telemetry via either Arcus technology or directly via Microsoft's `TelemetryClient` ([more info](https://observability.arcus-azure.net/Features/telemetry-enrichment)). +* `WriteTo.AzureApplicationInsightsWithConnectionString`: Serilog's sink that writes all the logged telemetry to Application Insights ([more info](https://observability.arcus-azure.net/Features/sinks/azure-application-insights)) +* `app.UseHttpCorrelation()`: retrieves the HTTP correlation from the incoming request or generates a new set (first request). This correlation information is set into the `IHttpCorrelationInfoAccessor`. ([more info](https://webapi.arcus-azure.net/features/correlation)). +* `app.UseRequestTracking()`: tracks the incoming HTTP request as a telemetry request. ([more info](https://webapi.arcus-azure.net/features/logging)). + +> ⚠ Note that when initializing the Application Insights Serilog sink, you should use the Arcus secret store to retrieve this connection string. Setting this up requires you to reload the logger after the application is build. For more information, see [this dedicated section](https://observability.arcus-azure.net/Features/sinks/azure-application-insights#q-where-can-i-initialize-the-logger-in-an-aspnet-core-application-or-other-hosted-service) that describes how to do this. + +> ⚠ Note that the order of the middleware component registrations is important. The HTTP request tracking needs the endpoint routing to figure out if the request should be tracked, for example. For more information on our different middleware components, see [our Web API feature documentation](https://webapi.arcus-azure.net/features/logging). + +With these HTTP correlation additions, we can easily put a message on the Service Bus queue, without any additional Arcus-related functionality: +```csharp +using Azure.Messaging.ServiceBus; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Azure; + +[ApiController] +[Route("api/v1/order")] +public class OrderController : ControllerBase +{ + private readonly ServiceBusSender _serviceBusSender; + + public OrderController(IAzureClientFactory clientFactory) + { + ServiceBusClient client = clientFactory.CreateClient("Order Worker"); + _serviceBusSender = client.CreateSender("orders"); + } + + [HttpPost] + public async Task Post([FromBody] ProductOrderRequest productRequest) + { + var order = new Order(productRequest); + + var data = BinaryData.FromObjectAsJson(order); + await _serviceBusSender.SendMessageAsync(data); + + return Accepted(); + } +} +``` + +### Order Worker: add Arcus functionality +The **Order Worker** didn't had any implementation, so let's add this now. We want to receive a message on an Azure Service Bus queue, and track that as a linked request in Application Insights. +We will be using the [Arcus message pump](https://messaging.arcus-azure.net/Features/message-pumps/service-bus) for this, as it allows us to focus solely on the message handling and does request tracking built-in. + +First, let's install these packages: +```shell +PM > Install-Package Arcus.Messaging.Pumps.ServiceBus -MinimumVersion 1.4.0 +PM > Install-Package Arcus.Observability.Telemetry.Serilog.Sinks.ApplicationInsights -MinimumVersion 2.7.0 +PM > Install-Package Serilog.Extensions.Hosting +``` + +> ⚡ Note that these additions are built-in into the [Arcus project templates](https://templates.arcus-azure.net/). + +These packages allows us to register the message pump and the Serilog logging: +```csharp +using Microsoft.Extensions.Azure; +using Microsoft.Extensions.DependencyInjection; +using Serilog; +using Serilog.Configuration; +using Serilog.Events; + +public class Program +{ + public static void Main(string[] args) + { + IHost host = + Host.CreateDefaultBuilder(args) + .ConfigureServices(services => + { + // [Add] Arcus + Microsoft component name/version registration + services.AddAppName("Order Worker"); + services.AddAssemblyAppVersion(); + + // [Add] Arcus message pump on Service Bus 'orders' queue + services.AddServiceBusQueueMessagePumpUsingManagedIdentity("orders", "") + // [Add] Arcus message handler that processes the received 'order' on the Service Bus queue + .WithServiceBusMessageHandler(); + }) + // [Add] Register the Serilog logger as the application's logger + .UseSerilog(Log.Logger) + .Build(); + + // [Add] Serilog configuration that writes to Application Insights + Log.Logger = new LoggerConfiguration() + .MinimumLevel.Information() + .MinimumLevel.Override("Microsoft", LogEventLevel.Information) + .Enrich.FromLogContext() + .Enrich.WithVersion(host.Services) + .Enrich.WithComponentName(host.Services) + .WriteTo.AzureApplicationInsightsWithConnectionString(host.Services, "") + .CreateLogger(); + + host.Run(); + } +} +``` + +Following additions are made: +* `AddAppName`/`Add...AppVersion`: configures for both Arcus & Microsoft the component's name and version when tracking telemetry via either Arcus technology or directly via Microsoft's `TelemetryClient` ([more info](https://observability.arcus-azure.net/Features/telemetry-enrichment)). +* `AddServiceBusQueueMessagePumpUsingManagedIdentity`: registers an Arcus message pump listening on an Azure Service Bus 'orders' queue. Received messages will automatically be tracked as Service Bus requests in Application Insights. + * `WithServiceBusMessageHandler`: registers a custom `OrderMessageHandler` to process the deserialized `Order` message. +* `WriteTo.AzureApplicationInsightsWithConnectionString`: Serilog's sink that writes all the logged telemetry to Application Insights ([more info](https://observability.arcus-azure.net/Features/sinks/azure-application-insights)) + +> ⚠ Note that when initializing the Application Insights Serilog sink, you should use the Arcus secret store to retrieve this connection string. Setting this up requires you to reload the logger after the application is build. For more information, see [this dedicated section](https://observability.arcus-azure.net/Features/sinks/azure-application-insights#q-where-can-i-initialize-the-logger-in-an-aspnet-core-application-or-other-hosted-service) that describes how to do this. + +The `OrderMessageHandler` is currently doing nothing: +```csharp +using Arcus.Messaging.Abstractions; +using Arcus.Messaging.Abstractions.ServiceBus; +using Arcus.Messaging.Abstractions.ServiceBus.MessageHandling; +using Microsoft.Extensions.Logging; + +public class OrderMessageHandler : IAzureServiceBusMessageHandler +{ + private readonly ILogger _logger; + + public OrderMessageHandler(ILogger logger) + { + _logger = logger; + } + + public async Task ProcessMessageAsync( + Order message, + AzureServiceBusMessageContext messageContext, + MessageCorrelationInfo correlationInfo, + CancellationToken cancellationToken) + { + _logger.LogInformation("Order processed!"); + } +} +``` + +> 💡 For more information on how Arcus handles message handling in message pumps, see [our dedicated messaging feature documentation](https://messaging.arcus-azure.net/Features/message-pumps/service-bus). + +### Run the solution +When both services are adapted, we can run the solution. Arcus makes sure that the link between the API and Service Bus is visible in Application Insights, without much effort from the consumer. Note that the `OrderMessageHandler` also receives a correlation model in its `ProcessMessageAsync` method which allows users to interact further with the passed-along correlation. + +Once the applications run, we send a request to the **Order API**, like: +```powershell +curl -Method POST ` + -Headers @{'Content-Type'='application/json'} ` + 'http://localhost:5000/api/v1/order' ` + -Body '{ "ProductName": "Fancy desk", "Amount": 3 }' + +// StatusCode : 202 +// Headers: : X-Transaction-ID=9d02c0f4782b45618181e84c4221b056 +// X-Operation-ID=f63bcab4b52b8373 +``` + +The real result, though, happens in Application Insights. + +The application map (`Application Insights > Investigate > Application Map`) shows a clear relationship between the two services: +![Product API <> Stock API application map example](/media/servicebus-worker-service-w3c-correlation-example-applicationmap.png) + +If you copy the `X-Transaction-ID` from the response (`585ea273-eaa9-44bb-905a-6805bd418566`) and past it in the transaction search (`Application Insights > Investigate > Transaction search`), you'll see this relationship in more detail when you select the initial HTTP request to the **Order API**. You clearly see how the initial request to the **Order API** is the caller of the dependency towards the **Order Worker**: +![Product API <> Stock API transaction search example](/media/servicebus-worker-service-w3c-correlation-example-transactionsearch.png) + +## Further reading +* [Arcus Service Bus messaging documentation](https://messaging.arcus-azure.net/Features/message-pumps/service-bus) +* [Arcus Application Insights Serilog sink documentation](https://observability.arcus-azure.net/Features/sinks/azure-application-insights) +* Messaging blogs + * [Taking messaging to the next level with Arcus Messaging v1.0](https://www.codit.eu/blog/taking-messaging-to-the-next-level-with-arcus-messaging-v1-0/) +* Web API blogs + * [Out-of-the-box Request Tracking, Simplified HTTP Correlation & JWT Authorization in Arcus Web API v1.0](https://www.codit.eu/blog/out-of-the-box-request-tracking-simplified-http-correlation-jwt-authorization-in-arcus-web-api-v1-0/) + * [Enhanced request tracking in Arcus Web API v1.3](https://www.codit.eu/blog/enhanced-request-tracking-in-arcus-web-api-v1-3/) +* Observability blogs + * [Announcing Arcus Observability](https://www.codit.eu/blog/announcing-arcus-observability/) + * [Measure a Variety of Azure Dependencies with Observability v0.2](https://www.codit.eu/blog/measure-a-variety-of-azure-dependencies-with-observability-v0-2/) + * [Service Correlation Preparation & .NET 6 Support in Arcus Observability v2.4](https://www.codit.eu/blog/service-correlation-preparation-net-6-support-in-arcus-observability-v2-4/) \ No newline at end of file diff --git a/docs/versioned_docs/version-v2.7.0/02-Guidance/Service-to-service Correlation/use-service-to-service-correlation-in-web-api.md b/docs/versioned_docs/version-v2.7.0/02-Guidance/Service-to-service Correlation/use-service-to-service-correlation-in-web-api.md new file mode 100644 index 00000000..bdd724ec --- /dev/null +++ b/docs/versioned_docs/version-v2.7.0/02-Guidance/Service-to-service Correlation/use-service-to-service-correlation-in-web-api.md @@ -0,0 +1,329 @@ +--- +title: "Use service-to-service correlation in Web API solutions" +layout: default +--- + +# Use service-to-service correlation in Web API solutions +The concept of service-to-service correlation is big and complex and spans multiple Arcus libraries. This user-guide will walk through all the available Arcus features that work together to facilitate service-to-service correlation in your Web API solutions. For a general introduction, see the [introduction page on service-to-service correlation](index.md). + +To use service-to-service correlation in Web API solutions, both the sending and receiving side of the separate API components have to be adapted. The HTTP correlation will be passed through from one service to another via the HTTP headers which the internal Arcus system will use to link your services together. + +The following diagram shows this communication more clearly: +![HTTP correlation diagram](/media/http-correlation-w3c.png) + +## How does this work? +Three kind of correlation ID's are used to create the relationship: +* **Transaction ID**: this ID is the one constant in the diagram. This ID is used to describe the entire transaction, from beginning to end. All telemetry will be tracked under this ID. +* **Operation ID**: this ID describes a single operation within the transaction. This ID is used within a service to link all telemetry of that service together. +* **Operation Parent ID**: this ID is create the parent/child link across services. When service A calls service B, then service A is the so called 'parent' of service B. + +The following list shows each step in the diagram: +1. The initial call in this example doesn't contain any correlation headers. This can be seen as a first interaction call to a service. +2. Upon receiving at service A, the application will generate new correlation information. This correlation info will be used when telemetry is tracked on the service. +3. When a call is made to service B, the **transaction ID** is sent but also the **operation parent ID** in the form of a hierarchical structure. +4. The `jkl` part of this ID, describes the new parent ID for service B (when service B calls service C, then it will use `jkl` as parent ID) +5. Service B responds to service A with the same information as the call to service B. +6. The user receives both the **transaction ID** and **operation parent ID** in their final response. + +> 💡 Additional configuration is available to tweak this functionality, see the [dedicated Arcus Web API feature documentation](https://webapi.arcus-azure.net/features/correlation) for more in-depth information on HTTP correlation. + +## API Demonstration +In this user-guide, two fictive API applications will be used to represent Service A and Service B of the diagram. Both services will be adapted to use service-to-service correlation. + +* **Product API** (Service A): receives an request to order a product if enough items of that product are in stock. +* **Stock API** (Service B): response with the available items in stock of a given product + +The user interacts with the **Product API** to order their product. Internally, the **Product API** will contact the **Stock API** to verify if there are enough items of that product to complete the order. + +> ⚠ Take into account that this sample should only be used for demonstration purposes and does not reflect a fully production-ready implementation. We have chosen to only provide the bare bones of the application so the changes are clear. + +### Product API: startup code +First, lets look at the initial code for the **Product API**. The startup code has the hosting and routing functionality to use API controllers. The configuration value `STOCK_API_URL` already reveals that this API will contact another API. This is also the reason why we added `.AddHttpClient()` to the startup code so we can inject `HttpClient` instances. ([More information on injecting HTTP clients in ASP.NET Core](https://docs.microsoft.com/en-us/aspnet/core/fundamentals/http-requests?view=aspnetcore-6.0)). +```csharp +public class Program +{ + public static void Main(string[] args) + { + WebApplicationBuilder builder = WebApplication.CreateBuilder(args); + builder.Services.AddRouting(); + builder.Services.AddControllers(); + + builder.Configuration.AddInMemoryCollection(new[] + { + new KeyValuePair("STOCK_API_URL", "http://localhost:788") + }); + builder.Services.AddHttpClient("Stock API"); + + WebApplication app = builder.Build(); + app.UseRouting(); + app.UseEndpoints(endpoints => endpoints.MapControllers()); + app.Run("http://localhost:787"); + } +} +``` + +The sole API controller in the **Product API** makes sure that we receive the product order request, contact the **Stock API** to request the available stock for that product, and determine the user response based on that availability. +```csharp +using System.Net.Http; +using System.Net.Http.Json; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Configuration; + +public class ProductOrderRequest +{ + public string ProductName { get; set; } + public int Amount { get; set; } +} + +[ApiController] +[Route("api/v1/product")] +public class ProductController : ControllerBase +{ + private readonly IHttpClientFactory _httpClientFactory; + private readonly IConfiguration _configuration; + + public ProductController( + IHttpClientFactory httpClientFactory, + IConfiguration configuration) + { + _httpClientFactory = httpClientFactory; + _configuration = configuration; + } + + [HttpPost] + public async Task Post([FromBody] ProductOrderRequest request) + { + HttpClient httpClient = _httpClientFactory.CreateClient("Stock API"); + + var endpoint = $"{_configuration["STOCK_API_URL"]}/api/v1/stock?productName={request.ProductName}"; + + var response = await httpClient.GetFromJsonAsync(endpoint); + if (response.AvailableItems >= request.Amount) + { + return Accepted(); + } + + return StatusCode(StatusCodes.Status500InternalServerError); + } +} +``` + +### Stock API: startup code +The startup code of the **Stock API** only has the hosting and routing functionality. Notice that this API is hosted on the same port that the **Product API** uses in its configuration. +```csharp +public class Program +{ + public static void Main(string[] args) + { + WebApplicationBuilder builder = WebApplication.CreateBuilder(args); + builder.Services.AddRouting(); + builder.Services.AddControllers(); + + WebApplication app = builder.Build(); + app.UseRouting(); + app.UseEndpoints(endpoints => endpoints.MapControllers()); + app.Run("http://localhost:788"); + } +} +``` + +The sole API controller in the **Stock API** returns a fictive amount of stock for a given product. +```csharp +using Microsoft.AspNetCore.Mvc; + +public class ProductStockResponse +{ + public string ProductName { get; set; } + public int AvailableItems { get; set; } +} + +[ApiController] +[Route("/api/v1/stock")] +public class StockController : ControllerBase +{ + [HttpGet] + public async Task Get([FromQuery] string productName) + { + var response = new ProductStockResponse + { + ProductName = productName, + AvailableItems = 3 + }; + + return Ok(response); + } +} +``` + +Right now, we haven't used any Arcus functionality, only built-in ASP.NET Core features. + +### Product API: add Arcus HTTP correlation +Now that we have explained the application code, we can add the Arcus functionality that will provide clear telemetry for the API interaction. + +First, these packages need to be installed: +```shell +PM > Install-Package Arcus.WebApi.Logging -MinimumVersion 1.7.0 +PM > Install-Package Arcus.Observability.Telemetry.Serilog.Sinks.ApplicationInsights -MinimumVersion 2.7.0 +``` + +> ⚡ Note that all these packages and additions are by default available in the [Arcus project templates](https://templates.arcus-azure.net/). + +> For more information on `Arcus.WebApi.Logging`, see [these dedicated feature docs](https://webapi.arcus-azure.net/features/logging), for more information on `Arcus.Observability.Telemetry.Serilog.Sinks.ApplicationInsights`, see [these dedicated feature docs](https://observability.arcus-azure.net/Features/sinks/azure-application-insights). + +The following additions have to be made to the startup code for the **Product API** to be able to track the incoming/outgoing HTTP requests, and to log to Application Insights. +```csharp +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; + +public class Program +{ + public static void Main(string[] args) + { + WebApplicationBuilder builder = WebApplication.CreateBuilder(args); + builder.Services.AddRouting(); + builder.Services.AddControllers(); + + // [Add] Makes HTTP correlation available throughout application + builder.Services.AddHttpCorrelation(); + + // [Add] Arcus + Microsoft component name/version registration + builder.Services.AddAppName("Product API"); + builder.Services.AddAssemblyAppVersion(); + + builder.Configuration.AddInMemoryCollection(new[] + { + new KeyValuePair("STOCK_API_URL", "http://localhost:788") + }); + builder.Services.AddHttpClient("Stock API"); + + // [Add] Serilog configuration that writes to Application Insights + builder.Host.UseSerilog((context, provider, config) => + { + config.MinimumLevel.Verbose() + .MinimumLevel.Override("Microsoft", LogEventLevel.Information) + .Enrich.WithComponentName(provider) + .Enrich.WithVersion(provider) + .Enrich.WithHttpCorrelationInfo(provider) + .WriteTo.AzureApplicationInsightsWithConnectionString(provider, ""); + }); + + WebApplication app = builder.Build(); + // [Add] Gets HTTP correlation from incoming request + app.UseHttpCorrelation(); + app.UseRouting(); + // [Add] Tracks incoming request + app.UseRequestTracking(); + app.UseEndpoints(endpoints => endpoints.MapControllers()); + app.Run("http://localhost:787"); + } +} +``` + +Following additions are made: +* `builder.Services.AddHttpCorrelation()`: adds the HTTP correlation services to the application services. This registers the `IHttpCorrelationInfoAccessor` that is used to get/set the HTTP correlation throughout the application. ([more info](https://webapi.arcus-azure.net/features/correlation)) +* `AddAppName`/`Add...AppVersion`: configures for both Arcus & Microsoft the component's name and version when tracking telemetry via either Arcus technology or directly via Microsoft's `TelemetryClient` ([more info](https://observability.arcus-azure.net/Features/telemetry-enrichment)). +* `WriteTo.AzureApplicationInsightsWithConnectionString`: Serilog's sink that writes all the logged telemetry to Application Insights ([more info](https://observability.arcus-azure.net/Features/sinks/azure-application-insights)) +* `app.UseHttpCorrelation()`: retrieves the HTTP correlation from the incoming request or generates a new set (first request). This correlation information is set into the `IHttpCorrelationInfoAccessor`. ([more info](https://webapi.arcus-azure.net/features/correlation)). +* `app.UseRequestTracking()`: tracks the incoming HTTP request as a telemetry request. ([more info](https://webapi.arcus-azure.net/features/logging)). + +> ⚠ Note that when initializing the Application Insights Serilog sink, you should use the Arcus secret store to retrieve this connection string. Setting this up requires you to reload the logger after the application is build. For more information, see [this dedicated section](https://observability.arcus-azure.net/Features/sinks/azure-application-insights#q-where-can-i-initialize-the-logger-in-an-aspnet-core-application-or-other-hosted-service) that describes how to do this. + +> ⚠ Note that the order of the middleware component registrations is important. The HTTP request tracking needs the endpoint routing to figure out if the request should be tracked, for example. For more information on our different middleware components, see [our Web API feature documentation](https://webapi.arcus-azure.net/features/logging). + +### Stock API: add Arcus HTTP correlation +The **Stock API** will need similar changes, but different from the **Product API** is that this API doesn't have to send correlated requests, only receive them. + +The same packages needs to be installed: +```shell +PM > Install-Package Arcus.WebApi.Logging -MinimumVersion 1.7.0 +PM > Install-Package Arcus.Observability.Telemetry.Serilog.Sinks.ApplicationInsights -MinimumVersion 2.7.0 +``` + +> ⚡ Note that all these packages and additions are by default available in the [Arcus project templates](https://templates.arcus-azure.net/). + +The new startup code is very similar. The only difference is missing HTTP client registration: +```csharp +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; + +public class Program +{ + public static void Main(string[] args) + { + WebApplicationBuilder builder = WebApplication.CreateBuilder(args); + builder.Services.AddRouting(); + builder.Services.AddControllers(); + + // [Add] Makes HTTP correlation available throughout application + builder.Services.AddHttpCorrelation(); + + // [Add] Arcus + Microsoft component name/version registration + builder.Services.AddAppName("Stock API"); + builder.Services.AddAssemblyAppVersion(); + + // [Add] Serilog configuration that writes to Application Insights + builder.Host.UseSerilog((context, provider, config) => + { + config.MinimumLevel.Verbose() + .MinimumLevel.Override("Microsoft", LogEventLevel.Information) + .Enrich.WithComponentName(provider) + .Enrich.WithVersion(provider) + .Enrich.WithHttpCorrelationInfo(provider) + .WriteTo.AzureApplicationInsightsWithConnectionString(provider, ""); + }); + + WebApplication app = builder.Build(); + // [Add] Gets HTTP correlation from incoming request + app.UseHttpCorrelation(); + app.UseRouting(); + // [Add] Tracks incoming request + app.UseRequestTracking(); + app.UseEndpoints(endpoints => endpoints.MapControllers()); + app.Run("http://localhost:788"); + } +} +``` + +> ⚠ Note that when initializing the Application Insights Serilog sink, you should use the Arcus secret store to retrieve this connection string. Setting this up requires you to reload the logger after the application is build. For more information, see [this dedicated section](https://observability.arcus-azure.net/Features/sinks/azure-application-insights#q-where-can-i-initialize-the-logger-in-an-aspnet-core-application-or-other-hosted-service) that describes how to do this. + +> ⚠ Note that the order of the middleware component registrations is important. The HTTP request tracking needs the endpoint routing to figure out if the request should be tracked, for example. For more information on our different middleware components, see [our Web API feature documentation](https://webapi.arcus-azure.net/features/logging). + +### Run the API solution +When both services are adapted to use Arcus' HTTP correlation, we can run the solution. It's worth noting that during these changes, we didn't once change the application code (controllers); only the startup code. This one of the many benefits from Arcus as it abstracts away all the infrastructure and boilerplate code and doesn't pollute application code with something infrastructure-like as service-to-service correlation. + +Once the applications run, we send a request to the **Product API**, like: +```powershell +curl -Method POST ` + -Headers @{'Content-Type'='application/json'} ` + 'http://localhost:787/api/v1/product' ` + -Body '{ "ProductName": "Fancy desk", "Amount": 3 }' + +// StatusCode : 202 +// Headers: : X-Transaction-ID=6fe3ca03cab168d171a4baa6bd965f7e +// X-Operation-ID=b8229991ef6c5fd4 +``` + +The real result, though, happens in Application Insights. + +The application map (`Application Insights > Investigate > Application Map`) shows a clear relationship between the two services: +![Product API <> Stock API application map example](/media/product-stock-api-service-correlation-example-applicationmap.png) + +If you copy the `X-Transaction-ID` from the response (`8a622fa4-a757-47a0-9db9-e22d00328087`) and past it in the transaction search (`Application Insights > Investigate > Transaction search`), you'll see this relationship in more detail when you select the initial HTTP request to the **Product API**. You clearly see how the initial request to the **Product API** is the caller of the dependency towards the **Stock API**: +![Product API <> Stock API transaction search example](/media/product-stock-api-service-w3c-correlation-example-transactionsearch.png) + +## Conclusion +In this user guide, you've seen how the Arcus HTTP correlation functionality can be used to set up service-to-service correlation in Web API solutions. The service-to-service correlation is a very wide topic and can be configured with many options. See the [this documentation page](https://webapi.arcus-azure.net/features/correlation) to learn more about HTTP correlation in Web API applications. + +## Further reading +* [Arcus Web API HTTP correlation documentation](https://webapi.arcus-azure.net/features/correlation) +* [Arcus Web API HTTP request tracking documentation](https://webapi.arcus-azure.net/features/logging) +* [Arcus Application Insights Serilog sink documentation](https://observability.arcus-azure.net/Features/sinks/azure-application-insights) +* Web API blogs + * [Out-of-the-box Request Tracking, Simplified HTTP Correlation & JWT Authorization in Arcus Web API v1.0](https://www.codit.eu/blog/out-of-the-box-request-tracking-simplified-http-correlation-jwt-authorization-in-arcus-web-api-v1-0/) + * [Enhanced request tracking in Arcus Web API v1.3](https://www.codit.eu/blog/enhanced-request-tracking-in-arcus-web-api-v1-3/) +* Observability blogs + * [Announcing Arcus Observability](https://www.codit.eu/blog/announcing-arcus-observability/) + * [Measure a Variety of Azure Dependencies with Observability v0.2](https://www.codit.eu/blog/measure-a-variety-of-azure-dependencies-with-observability-v0-2/) + * [Service Correlation Preparation & .NET 6 Support in Arcus Observability v2.4](https://www.codit.eu/blog/service-correlation-preparation-net-6-support-in-arcus-observability-v2-4/) + * [Service-to-Service Correlation, One of the Biggest Arcus Features in Observability v2.5](https://www.codit.eu/blog/service-to-service-correlation-one-of-the-biggest-arcus-features-in-observability-v2-5/) diff --git a/docs/versioned_docs/version-v2.7.0/02-Guidance/use-with-dotnet-and-aspnetcore.md b/docs/versioned_docs/version-v2.7.0/02-Guidance/use-with-dotnet-and-aspnetcore.md new file mode 100644 index 00000000..d3da3649 --- /dev/null +++ b/docs/versioned_docs/version-v2.7.0/02-Guidance/use-with-dotnet-and-aspnetcore.md @@ -0,0 +1,158 @@ +--- +title: "Using Arcus & Serilog in ASP.NET Core" +layout: default +--- + +# Using Arcus & Serilog in ASP.NET Core +The Arcus Application Insights Serilog sink is a great way to simplify telemetry tracking and application logging. Unlike Microsoft's `TelemetryClient` for telemetry tracking, Arcus uses the common `ILogger` infrastructure to track and link telemetry in Application Insights. + +This user guide will walk through the steps to configure the Arcus Application Insights Serilog sink in ASP.NET Core applications. + +## Installation +The example in this user guide uses following packages. + +```shell +PM > Install-Package -Name Serilog.AspNetCore +PM > Install-Package -Name Arcus.Observability.Telemetry.Serilog.Sinks.ApplicationInsights +PM > Install-Package -Name Arcus.Security.Providers.AzureKeyVault +PM > Install-Package -Name Arcus.WebApi.Logging +``` + +### Serilog.AspNetCore +Since Arcus uses Serilog, we expect you to setup Serilog as your logging infrastructure. This is also Microsoft's recommended logging system ([more info](https://serilog.net/)). + +### Arcus.Observability.Telemetry.Serilog.Sinks.ApplicationInsights +The Arcus Application Insights Serilog sink builds on top of Serilog and acts as a bridge between the common `ILogger` infrastructure and the Application Insights telemetry. + +### Arcus.Security.Providers.AzureKeyVault +It is recommended to configure the Arcus Application Insights Serilog sink with the [Application Insights connection string](https://docs.microsoft.com/en-us/azure/azure-monitor/app/sdk-connection-string). This connection string should be safely stored. This example uses the Arcus secret store with Azure Key Vault integration ([more info](https://security.arcus-azure.net/features/secret-store)). + +### Arcus.WebApi.Logging +As this user guide shows how a fully working API application is set up for Application Insights tracking, we will also use `Arcus.WebApi.Logging`. The Arcus WebApi library builds on top of Arcus Observability, specific for API applications. This will make sure that the application correlation is stored in the HTTP context during send/receive operations ([more info](https://webapi.arcus-azure.net/)). + +## Setting up Serilog with Arcus +The following shows a complete code sample of how Arcus and Serilog are configured together. Each critical point is explained afterwards. In short, what's happening is that Serilog is first configured as a basic debug logger so that startup failures are logged to the console (1). After the application is build, the 'real' Serilog setup is configured that uses the Arcus Application Insights Serilog sink (6) with the connection string that is stored in the Arcus secret store (3). + +```csharp +using Microsoft.AspNetCore.Builder; +using Microsoft.DependencyInjection.Extensions; +using Serilog; +using Serilog.Configuration; + +public class Program +{ + public static async Task Main(string[] args) + { + // 1. Configure startup Serilog logger. + Log.Logger = new LoggerConfiguration() + .MinimumLevel.Debug() + .WriteTo.Console() + .CreateBootstrapLogger(); + + try + { + WebApplicationBuilder builder = WebApplication.CreateBuilder(args); + builder.Services.AddRouting(); + builder.Services.AddControllers(); + // 2. Register the HTTP correlation services to access the current correlation information during the HTTP request processing. + builder.Services.AddHttpCorrelation(); + + // 3. Configure Arcus secret store with Application Insights connection string (more info on secret store: https://security.arcus-azure.net/features/secret-store). + builder.Host.ConfigureSecretStore((context, stores) => stores.AddAzureKeyVaultWithManagedServiceIdentity(...)); + + // 4. Use Serilog's static Logger as application logger. + builder.Host.UserSerilog(Log.Logger); + + WebApplication app = builder.Build(); + // 5. Use the Arcus HTTP correlation middleware to set the correlation information on received HTTP requests. + app.UseHttpCorrelation(); + app.UseRouting(); + app.UseEndpoints(endpoints => endpoints.MapControllers()); + + await ConfigureSerilogAsync(app); + await app.RunAsync("http://localhost:5000"); + + return 0; + } + catch (Exception exception) + { + Log.Fatal(exception, "Host terminated unexpectedly"); + return 1; + } + finally + { + Log.CloseAndFlush(); + } + } + + // 6. Retrieve Application Insights connection string to configure the 'real' application logger. + private static async Task ConfigureSerilogAsync(WebApplication app) + { + var secretProvider = app.Services.GetRequiredService(); + string connectionString = await secretProvider.GetRawSecretAsync("APPLICATIONINSIGHTS_CONNECTION_STRING"); + + var reloadLogger = (ReloadableLogger) Log.Logger; + reloadLogger.Reload(config => + { + config.ReadFrom.Configuration(app.Configuration) + .MinimumLevel.Override("Microsoft", LogEventLevel.Warning) + .Enrich.FromLogContext() + .Enrich.WithVersion() + .Enrich.WithComponentName("API") + .Enrich.WithHttpCorrelationInfo(app.Services) + .WriteTo.Console() + .WriteTo.AzureApplicationInsightsWithConnectionString(connectionString); + + return config; + }); + } +} +``` + +1. Before the application is build, we should set up a basic Serilog logger to log any startup failures. That's what the `try/catch/finally` block is all about. This is a safe and reliable way to setup Serilog ([more info](https://github.com/serilog/serilog-aspnetcore)). Note that the `.CreateBootstrapLogger()` means that the Serilog logger can be 'reloaded' afterwards. +2. The connection string to contact the Application Insights resource should be stored safely. The `.ConfigureSecretStore` will register a composite `ISecretProvider` interface to interact with all the registered secret providers ([more info](https://security.arcus-azure.net/features/secret-store)). +3. To make sure that any written telemetry to Application Insights is correlated, the Serilog sink should be able to access the current HTTP correlation of the received request. The `.AddHttpCorrelation()` makes sure that the HTTP correlation services are available. +4. Make sure that Serilog uses the static configured logger as application logger that gets injected as `ILogger` in your application ([more info](https://github.com/serilog/serilog-aspnetcore)). +5. To set the current HTTP correlation information of the received HTTP request, the Arcus HTTP correlation middleware has to be registered with `.UseHttpCorrelation()`. This will make sure that the HTTP correlation information will be set, using the previously registered HTTP correlation services (3). +6. When the application is build, the 'real' Serilog configuration can be set up. This will extract the Application Insights connection string from the Arcus secret store and use it to configure the Arcus Serilog sink, using `AzureApplicationInsightsWithConnectionString`. This example also adds `WithHttpCorrelationInfo` from the `Arcus.WebApi.Logging` library to include the HTTP correlation ([more info](https://webapi.arcus-azure.net/features/correlation)). + +> 💡 Note: this setup is by default available in the [Arcus web API project template](https://templates.arcus-azure.net/features/web-api-template). + +## Writing telemetry with ILogger +Once Serilog is set up, you can write telemetry data with the general `ILogger` instance injected in your application. This example uses the Arcus logger extension to track custom events in Application Insights, but there is a lot more that can be tracked. See [our list of telemetry types](../03-Features/writing-different-telemetry-types.md) to find out all the available types that can be written with Arcus. + +```csharp +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; + +[ApiController] +[Route("api/v1/order")] +public class OrderController : ControllerBase +{ + private readonly ILogger _logger; + + public OrderController(ILogger logger) + { + _logger = logger; + } + + [HttpPost] + public IActionResult Post([FromBody] Order order) + { + // Use logger for general logging: results in 'trace' in Application Insights. + _logger.LogInformation("Order {Id} processed!", order.Id); + + var contextualInformation = new Dictionary + { + {"Order ID", order.Id}, + {"Customer", order.Customer.Name} + }; + + // Use logger for telemetry tracking: results in 'custom event' in Application Insights. + _logger.LogCustomEvent("Order processed", contextualInformation); + + return Accepted(); + } +} +``` \ No newline at end of file diff --git a/docs/versioned_docs/version-v2.7.0/02-Guidance/use-with-dotnet-and-functions.md b/docs/versioned_docs/version-v2.7.0/02-Guidance/use-with-dotnet-and-functions.md new file mode 100644 index 00000000..a9b86008 --- /dev/null +++ b/docs/versioned_docs/version-v2.7.0/02-Guidance/use-with-dotnet-and-functions.md @@ -0,0 +1,123 @@ +--- +title: "Using Arcus & Serilog in Azure Functions (in-process)" +layout: default +--- + +# Using Arcus & Serilog in Azure Functions (in-process) +The Arcus Application Insights Serilog sink is a great way to simplify telemetry tracking and application logging. Unlike Microsoft's `TelemetryClient` for telemetry tracking, Arcus uses the common `ILogger` infrastructure to track and link telemetry in Application Insights. + +This user guide will walk through the steps to configure the Arcus Application Insights Serilog sink in an Azure Functions (in-process) HTTP trigger application. + +## Installation +The example in this user guide uses following packages. + +```shell +PM > Install-Package -Name Serilog.Extensions.Logging +PM > Install-Package -Name Serilog.Sinks.Console +PM > Install-Package -Name Arcus.Observability.Telemetry.AzureFunctions +PM > Install-Package -Name Arcus.Observability.Telemetry.Serilog.Sinks.ApplicationInsights +PM > Install-Package -Name Arcus.WebApi.Logging.AzureFunctions +``` + +### Serilog.AspNetCore +Since Arcus uses Serilog, we expect you to setup Serilog as your logging infrastructure. This is also Microsoft's recommended logging system ([more info](https://serilog.net/)). + +### Arcus.Observability.Telemetry.Serilog.Sinks.ApplicationInsights +The Arcus Application Insights Serilog sink builds on top of Serilog and acts as a bridge between the common `ILogger` infrastructure and the Application Insights telemetry. + +### Arcus.WebApi.Logging +As this user guide shows how a fully working API application is set up for Application Insights tracking, we will also use `Arcus.WebApi.Logging`. The Arcus WebApi library builds on top of Arcus Observability, specific for API applications. This will make sure that the application correlation is stored in the HTTP context during send/receive operations ([more info](https://webapi.arcus-azure.net/)). + +## Setting up Serilog with Arcus +The following shows a complete code sample of how Arcus and Serilog are configured together. Each critical point is explained afterwards. In short, what's happening is that (2) the Serilog configuration is built up to use Arcus Application Insights Serilog sink, (3) any default Application Insights interaction is removed, and (4) the Serilog configuration is registered in the application. + +> ⚠ Make sure to remove the `Logging` section from the `appsettings.json` *(if applicable)* as this will not be used by Serilog + +```csharp +using Microsoft.Azure.Functions.Extensions.DependencyInjection; + +[assembly: FunctionsStartup(typeof(Startup))] + +public class Startup : FunctionsStartup +{ + public override void Configure(IFunctionsHostBuilder builder) + { + // 1. Register the HTTP correlation services to access the current correlation information during the HTTP request processing. + builder.AddHttpCorrelation(); + + // 2. Create Serilog logger configuration model that writes to Application Insights. + IConfiguration appConfig = builder.GetContext().Configuration; + var connectionString = appConfig.GetValue("APPLICATIONINSIGHTS_CONNECTION_STRING"); + var logConfig = new LoggerConfiguration() + .MinimumLevel.Information() + .MinimumLevel.Override("Microsoft", LogEventLevel.Information) + .Enrich.FromLogContext() + .Enrich.WithComponentName("Azure HTTP Trigger") + .Enrich.WithVersion() + .WriteTo.Console() + .WriteTo.AzureApplicationInsightsWithConnectionString(connectionString); + + builder.Services.AddLogging(logging => + { + // 3. Remove Microsoft's default Application Insights logger provider. + logging.RemoveMicrosoftApplicationInsightsLoggerProvider() + // 4. Use Serilog's logger configuration to create the application logger. + .AddSerilog(logConfig.CreateLogger(), dispose: true); + }); + } +} +``` + +1. To make sure that any written telemetry to Application Insights is correlated, the Serilog sink should be able to access the current HTTP correlation of the received request. The `.AddHttpCorrelation()` makes sure that the HTTP correlation services are available. +2. Serilog configuration can be set up. This will extract the Application Insights connection string from the Arcus secret store and use it to configure the Arcus Serilog sink, using `AzureApplicationInsightsWithConnectionString`. This example doesn't have any HTTP correlation enrichment by-default, as the services container is not build at this moment. For correlated +3. We need to call `RemoveMicrosoftApplicationInsightsLoggerProvider` to remove Microsoft's `ApplicationInsightsLoggerProvider` because it would conflict with our own Serilog Application Insights sink. We can't guarantee stable telemetry if Microsoft's logger provider is registered as this provider manipulates the telemetry before it get's send out to Application Insights. Removing it ensure that Arcus is in full control of the send-out telemetry. +4. Register the created Serilog configuration as the application logger. + +> 💡 Note: this setup is by default available in the [Arcus Azure Functions HTTP trigger project template](https://templates.arcus-azure.net/features/azurefunctions-http-template). + +## Writing telemetry with ILogger +Once Serilog is set up, you can write telemetry data with the general `ILogger` instance injected in your application. This example uses the Arcus logger extension to multi-dimensional metrics in Application Insights, but there is a lot more that can be tracked. See [our list of telemetry types](../03-Features/writing-different-telemetry-types.md) to find out all the available types that can be written with Arcus. + +```csharp +using Microsoft.Azure.Databricks.Client; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; + +public class DockerHubMetricScraperFunction +{ + private readonly DockerHubClient _dockerHubClient; + private readonly IConfiguration _configuration; + private readonly ILogger _logger; + + public DockerHubMetricScraperFunction(DockerHubClient dockerHubClient, IConfiguration configuration, ILogger logger) + { + Guard.NotNull(dockerHubClient, nameof(dockerHubClient)); + Guard.NotNull(configuration, nameof(configuration)); + Guard.NotNull(logger, nameof(logger)); + + _dockerHubClient = dockerHubClient; + _configuration = configuration; + _logger = logger; + } + + [FunctionName("docker-hub-metric-scraper")] + public async Task Run([TimerTrigger("0 */15 * * * *")] TimerInfo timer) + { + _logger.LogInformation($"Starting to scrape Docker Hub metrics at {DateTime.UtcNow}"); + + var repoName = _configuration["DOCKER_HUB_REPO_NAME"]; + var imageName = _configuration["DOCKER_HUB_IMAGE_NAME"]; + + var pullCount = await _dockerHubClient.GetImageMetricsAsync(repoName, imageName); + + var contextualInformation = new Dictionary + { + {"Repo Name", repoName}, + {"Image Name", imageName}, + {"Image ID", $"{repoName}/{imageName}"} + }; + + _logger.LogCustomMetric("Image Pulls", pullCount, contextualInformation); + } +} +``` \ No newline at end of file diff --git a/docs/versioned_docs/version-v2.7.0/03-Features/correlation.md b/docs/versioned_docs/version-v2.7.0/03-Features/correlation.md new file mode 100644 index 00000000..b3b5e262 --- /dev/null +++ b/docs/versioned_docs/version-v2.7.0/03-Features/correlation.md @@ -0,0 +1,109 @@ +--- +title: "Correlation" +layout: default +--- + +# Correlation + +`CorrelationInfo` provides a common set of correlation levels: + +- Transaction Id - ID that relates different requests together into a functional transaction. +- Operation Id - Unique ID information for a single request. +- Operation parent Id - ID of the original service that initiated the request. + +## Installation + +This feature requires to install our NuGet package + +```shell +PM > Install-Package Arcus.Observability.Correlation +``` + +## What We Provide + +The `Arcus.Observability.Correlation` library provides a way to get access to correlation information across your application. +What it **DOES NOT** provide is how this correlation information is initially set. + +It uses the Microsoft dependency injection mechanism to register an `ICorrelationInfoAccessor` and `ICorrelationInfoAccessor<>` implementation that is available. + +**Example** + +```csharp +using Arcus.Observability.Correlation; + +namespace Application +{ + public class Startup + { + public void ConfigureServices(IServiceCollection services) + { + // Adds operation and transaction correlation to the application, + // using the `DefaultCorrelationInfoAccessor` as `ICorrelationInfoAccessor` that stores the `CorrelationInfo` model internally. + services.AddCorrelation(); + } + } +} +``` +## Custom Correlation + +We register two interfaces during the registration of the correlation: `ICorrelationInfoAccessor` and `ICorrelationInfoAccessor<>`. +The reason is because some applications require a custom `CorrelationInfo` model, and with using the generic interface `ICorrelationInfoAccessor<>` we can support this. + +**Example** + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Arcus.Observability.Correlation; + +namespace Application +{ + public class OrderCorrelationInfo : CorrelationInfo + { + public string OrderId { get; } + } + + public class Startup + { + public void ConfigureService(IServiceCollection services) + { + services.AddCorrelation(); + } + } +} +``` + +## Accessing Correlation Throughout the Application + +When a part of the application needs access to the correlation information, you can inject one of the two interfaces: + +```csharp +using Arcus.Observability.Correlation; + +namespace Application +{ + public class OrderService + { + public OrderService(ICorrelationInfoAccessor accessor) + { + CorrelationInfo correlationInfo = accessor.CorrelationInfo; + } + } +} +``` + +Or, alternatively when using custom correlation: + +```csharp +using Arcus.Observability.Correlation; + +namespace Application +{ + public class OrderService + { + public OrderService(ICorrelationInfoAccessor accessor) + { + OrderCorrelationInfo correlationInfo = accessor.CorrelationInfo; + } + } +} +``` \ No newline at end of file diff --git a/docs/versioned_docs/version-v2.7.0/03-Features/making-telemetry-more-powerful.md b/docs/versioned_docs/version-v2.7.0/03-Features/making-telemetry-more-powerful.md new file mode 100644 index 00000000..f592a7e3 --- /dev/null +++ b/docs/versioned_docs/version-v2.7.0/03-Features/making-telemetry-more-powerful.md @@ -0,0 +1,64 @@ +--- +title: "Making telemetry more powerful" +layout: default +--- + +# Making telemetry more powerful + +## Providing contextual information + +In order to make telemetry more powerful we **highly recommend providing contextual information around what the situation is of your application**. That's why every telemetry type that you can write, allows you to provide context in the form of a dictionary. + +```csharp +using Microsoft.Extensions.Logging; + +// Provide context around event +var telemetryContext = new Dictionary +{ + {"Customer", "Arcus"}, + {"OrderId", "ABC"}, +}; + +logger.LogCustomEvent("Order Created", telemetryContext); +// Output: "Events Order Created (Context: [Customer, Arcus], [OrderId, ABC])" +``` + +By doing so, you'll be able to interact more efficient with your logs by filtering, searching, ... on it. + +We support this for all [telemetry types that you can write](./writing-different-telemetry-types.md). + +### Seeing the power in action + +Let's use an example - When measuring a metric you get an understanding of the count, in our case the number of orders received: + +```csharp +using Microsoft.Extensions.Logging; + +logger.LogCustomMetric("Orders Received", 133); +// Log output: "Metric Orders Received: 133 (Context: )" +``` + +If we output this to Azure Application Insights as a metric similar to our example: +![Single-dimension Metric](/media/single-dimensional-metric.png) + +However, you can very easily provide additional context, allowing you to get an understanding of the number of orders received and annotate it with the vendor information. + +```csharp +using Microsoft.Extensions.Logging; + +var telemetryContext = new Dictionary +{ + { "Customer", "Contoso"}, +}; + +logger.LogCustomMetric("Orders Received", 133, telemetryContext); +// Log output: "Metric Orders Received: 133 (Context: [Customer, Contoso])" +``` + +The outputted telemetry will contain that information and depending on the sink that you are using it's even going to be more powerful. + +For example, when using Azure Application Insights your metric will evolve from a single-dimensional metric to multi-dimensional metrics allowing you to get the total number of orders, get the number of orders per vendor or filter the metric to one specific vendor. + +Here we are using our multi-dimensional metric and splitting it per customer to get more detailed insights: + +![Multi-dimension Metric](/media/multi-dimensional-metrics.png) diff --git a/docs/versioned_docs/version-v2.7.0/03-Features/sinks/azure-application-insights.md b/docs/versioned_docs/version-v2.7.0/03-Features/sinks/azure-application-insights.md new file mode 100644 index 00000000..f879e8bc --- /dev/null +++ b/docs/versioned_docs/version-v2.7.0/03-Features/sinks/azure-application-insights.md @@ -0,0 +1,230 @@ +--- +title: "Azure Application Insights Sink" +layout: default +--- + +# Azure Application Insights Sink + +## What is it? + +The Azure Application Insights sink is an extension of the [official Application Insights sink](https://www.nuget.org/packages/Serilog.Sinks.ApplicationInsights/) that allows you to not only emit traces or events, but the whole Application Insights suite of telemetry types - Traces, Dependencies, Events, Requests & Metrics. + +You can easily configure the sink by providing the Azure Application Insights connection string or instrumentation key, but the connection string is preferred. + +```csharp +using Serilog; +using Serilog.Configuration; + +ILogger logger = new LoggerConfiguration() + .MinimumLevel.Debug() + .WriteTo.AzureApplicationInsightsWithConnectionString("") + .CreateLogger(); + +ILogger logger = new LoggerConfiguration() + .MinimumLevel.Debug() + .WriteTo.AzureApplicationInsightsWithInstrumentationKey("") + .CreateLogger(); +``` + +Alternatively, you can override the default minimum log level to reduce amount of telemetry being tracked : + +```csharp +using Serilog; +using Serilog.Configuration; + +ILogger logger = new LoggerConfiguration() + .MinimumLevel.Debug() + .WriteTo.AzureApplicationInsightsWithConnectionString("", restrictedToMinimumLevel: LogEventLevel.Warning) + .CreateLogger(); +``` + +> ⚡ When you want to re-use an already registered `TelemetryClient`, pass in the `IServiceProvider` to the method that registers the Application Insights sink. `TelemetryClient`s are automatically registered behind the scenes when using W3C correlation in Arcus Web API middleware (>= v1.7) or Arcus Messaging (>= v1.4). + +```csharp +using Serilog; +using Serilog.Configuration; + +ILogger logger = new LoggerConfiguration() + .MinimumLevel.Debug() + .WriteTo.AzureApplicationInsightsWithConnectionString(serviceProvider, "", restrictedToMinimumLevel: LogEventLevel.Warning) + .CreateLogger(); +``` + +For more information on this sink: [see this section](#azure-application-insights-sink). + +## Installation + +This feature requires to install our NuGet package + +```shell +PM > Install-Package Arcus.Observability.Telemetry.Serilog.Sinks.ApplicationInsights +``` + +## Configuration + +The Azure Application Insights sink has some additional configuration which can be changed to influence the tracking. + +### Requests + +### Request ID + +When tracking requests, the ID for the request telemetry is by default a generated GUID. The generation of this ID can be configured via the options. +This is useful (for example) in a service-to-service correlation system where you want the ID of the incoming request to be based on the sending system, or you want to incorporate the operation ID in the request ID. + +```csharp +using Serilog; +using Serilog.Configuration; + +ILogger logger = new LoggerConfiguration() + .WriteTo.AzureApplicationInsightsWithConnectionString("", options => + { + // Configurable generation function for the telemetry request ID. + options.Request.GenerateId = () => $"my-custom-ID-{Guid.NewGuid()}"; + }) +``` + +### Exceptions + +#### Properties + +When tracking exceptions, one can opt-in to track all the public properties of the exception which will be included as [custom dimensions](https://docs.microsoft.com/en-us/azure/azure-monitor/app/api-custom-events-metrics#custom-measurements-and-properties-in-analytics). +These public properties are formatted with the following pattern: `Exception-{0}` where `{0}` is the place where the public property's name is inserted. + +The value of the property will be the value of the custom dimension so that the custom dimension will be in the form `"Exception-{your-property-name}" = "your-property-value"`. + +This property format pattern can be configured, like shown in the following example: + +```csharp +using Serilog; +using Serilog.Configuration; + +ILogger logger = new LoggerConfiguration() + .WriteTo.AzureApplicationInsightsWithConnectionString("", options => + { + // Opt-in to track all the first-level exception properties; inherited properties will not be included. + options.Exception.IncludeProperties = true; + + // Property format to track the exception's properties (default: `"Exception-{0}"`) + options.Exception.PropertyFormat = "CustomException.{0}"); + }) + .CreateLogger(); +``` + +Let's take a custom exception to demonstrate: + +```csharp +public class OrderingException : Exception +{ + ... + + // Property to be included in Application Insights. + public int OrderNumber { get; } +} +``` + +With this configuration, the custom dimension name for the public properties of this exception will look like this: `"CustomException.OrderNumber"`. + +> ⚠️ The property format should be in the correct form in order to be used. It's passed to the `String.Format` eventually which will try to insert the exception's property name. + +### Correlation + +When tracking telemetry, the correlation information is enriched via the [Arcus Serilog correlation enrichment](../telemetry-enrichment.md). +This Serilog enricher is capable of customizing the log properties where the correlation will be added. When such custom setup is used, the same custom property names should be used here so that the Application Insights sink knows where to get the correlation information from. + +```csharp +string operationIdPropertyName = "MyOperationPropertyId"; +string transactionIdPropertyName = "MyTransactionPropertyId"; +string operationParentPropertyName = "MyOperationParentPropertyId"; +IServiceProvider serviceProvider = ... + +ILogger logger = new LoggerConfiguration() + .Enrich.WithCorrelationInfo(serviceProvider, options => + { + options.Correlation.OperationIdPropertyName = operationIdPropertyName; + options.Correlation.TransactionIdPropertyName = transactionIdPropertyName; + options.Correlation.OperationParentIdPropertyName = operationParentIdPropertyName; + }) + .WriteTo.AzureApplicationInsightsWithConnectionString("", options => + { + // Sets a custom property name where the operation ID should be added (default: 'OperationId'). + options.Correlation.OperationIdPropertyName = operationIdPropertyName; + + // Sets a custom property name where the transaction ID should be added (default: 'TransactionId'); + options.Correlation.TransactionIdPropertyName = transactionIdPropertyName; + + // Sets a custom property name where the operation parent ID should be added (default: 'OperationParentId'). + options.Correlation.OperationParentIdPropertyName = operationParentIdPropertyName; + }); +``` + +For more information on the way how Arcus handles correlation, see [this dedicated page](../correlation.md). + +## FAQ + +### Q: Why do I have to configure an instrumentation key (connection string) when using the Application Insights sink + +While the native Azure Application Insights SDK does not enforce an instrumentation key we have chosen to make it mandatory to provide one. + +By doing this, we allow you to fail fast and avoid running your application with a misconfigured telemetry setup. If it would be optional, you could have it running for days/weeks only to notice you are not sending any telemetry when everything is on fire and you are in the dark. + +If you want to optionally use our sink when there is an instrumentation key, we recommend using this simple pattern: + +```csharp +using Serilog; +using Serilog.Configuration; + +var loggerConfig = new LoggerConfiguration() + .MinimumLevel.Debug(); + +if (!string.IsNullOrWhiteSpace(key)) +{ + loggerConfig.WriteTo.AzureApplicationInsightsWithInstrumentationKey(key); +} + +ILogger logger = loggerConfig.CreateLogger(); +``` + +### Q: Where can I initialize the logger in an ASP.NET Core application or other hosted service? + +If the connection string is stored as a secret in -for instance- Azure KeyVault, the `ISecretProvider` from [Arcus secret store](https://security.arcus-azure.net/features/secret-store) can be used to retrieve the connection string. + +Use the `UseSerilog` extension method on `IHostBuilder` which accepts an `ILogger` and use the Serilog's static `Log.Logger` property to setup the logger. +This allows the Startup code to have preliminary logging before we can setup logging to Application Insights. This temporary logger is used during the execution of the startup code, including the secret store retrieval for the Application Insights connection string. Setting up the preliminary logger as a 'bootstrap' logger allows to reload the logger after we've set up logging to Application Insights. + +```csharp +using Serilog; +using Serilog.Configuration; +using Arcus.Security.Core; + +// Setup temporary logger to log any setup errors. +Log.Logger = new LoggerConfiguration() + .MinimumLevel.Debug() + .WriteTo.Console() + .CreateBootstrapLogger(); + +// Build application. +IHost host = + Host.CreateDefaultBuilder() + .ConfigureSecretStore((context, config, builder) => + { + // Configure the secret store here. + // See: https://security.arcus-azure.net/features/secret-store/ + }) + .UseSerilog(Log.Logger) + .Build(); + +// Initialize Application Insights. +var secretProvider = host.Services.GetRequiredService(); +string connectionString = await secretProvider.GetRawSecretAsync("APPLICATIONINSIGHTS_CONNECTION_STRING"); + +var reloadLogger = (ReloadableLogger) Log.Logger; +reloadLogger.Reload(config => +{ + return config.WriteTo.AzureApplicationInsightsWithConnectionString(host.Services, connectionString); +}); + +// Start application. +await host.RunAsync(); +``` + +This approach is by default used in the [Arcus templates](https://templates.arcus-azure.net/) so you don't have to set this up yourself. \ No newline at end of file diff --git a/docs/versioned_docs/version-v2.7.0/03-Features/telemetry-enrichment.md b/docs/versioned_docs/version-v2.7.0/03-Features/telemetry-enrichment.md new file mode 100644 index 00000000..f68b67f8 --- /dev/null +++ b/docs/versioned_docs/version-v2.7.0/03-Features/telemetry-enrichment.md @@ -0,0 +1,344 @@ +--- +title: "Telemetry Enrichment" +layout: default +--- + +# Telemetry Enrichment +We provide a variety of enrichers for Serilog + +## Installation +This feature requires to install our NuGet package + +```shell +PM > Install-Package Arcus.Observability.Telemetry.Serilog.Enrichers +``` + +## Application Enricher +The `Arcus.Observability.Telemetry.Serilog.Enrichers` library provides a [Serilog enricher](https://github.com/serilog/serilog/wiki/Enrichment) +that adds the application's component name to the log event as a log property with the name `ComponentName` and gives the opportunity to choose the location from where the application 'instance' should be retrieved. + +**Example** +Name: `ComponentName` +Value: `My application component` + +**Usage** + +```csharp +using Serilog; + +ILogger logger = new LoggerConfiguration() + .Enrich.WithComponentName("My application component") + .CreateLogger(); + +logger.Information("Some event"); +// Output: Some event {ComponentName: My application component, MachineName: MyComputer} +``` + +Or, alternatively one can choose to use the Kubernetes information which our [Application Insights](./sinks/azure-application-insights.md) sink will prioritize above the `MachineName` when determining the telemetry `Cloud.RoleInstance`. + +```csharp +using Serilog; + +ILogger logger = new LoggerConfiguration() + .Enrich.WithComponentName("My application component") + .Enrich.WithKubernetesInfo() + .CreateLogger(); + +logger.Information("Some event"); +// Output: Some event {ComponentName: My application component, MachineName: MachineName: MyComputer, PodName: demo-app} +``` + +### Automatic Application Insights `Cloud.RoleName` assignment in `TelemetryClient` +When tracking telemetry in Application Insights via the regular `TelemetryClient`, the application enrichment can be configured to use an `IAppName` registered instance so that both the Serilog enrichment as the Microsoft tracking can benefit from this component name assignment. + +First, the `IAppName` should be configured in the application services: +```csharp +using Microsoft.Extensions.DependencyInjection; + +public void ConfigureServices(IServiceCollection services) +{ + services.AddAppName("My application component"); +} +``` + +Afterwards, when the application enricher is added, you can pass in the `IServiceProvider` so that the application name is assigned via the `IAppName` instance: +```csharp +using Microsoft.Extensions.DependencyInjection; +using Serilog; + +IServiceProvider provider = ... +ILogger logger = new LoggerConfiguration() + .Enrich.WithComponentName(provider) + .CreateLogger(); +``` + +This setup will make sure that both for Arcus tracking as well as for Microsoft tracking, the same component name will be used. + +### Custom Serilog property names +The application enricher allows you to specify the name of the log property that will be added to the log event during enrichment. +By default this is set to `ComponentName`. + +```csharp +using Serilog; + +ILogger logger = new LoggerConfiguration() + .Enrich.WithComponentName( + componentName: "My application component", + propertyName: "MyComponentName") + .CreateLogger(); + +logger.Information("Some event"); +// Output: Some event {MyComponentName: My application component, MachineName: MyComputer} +``` + +## Correlation Enricher +The `Arcus.ObservabilityTelemetry.Serilog.Enrichers` library provides a [Serilog enricher](https://github.com/serilog/serilog/wiki/Enrichment) +that adds the `CorrelationInfo` information from the current context as log properties with the names `OperationId` and `TransactionId`. + +You can use your own `ICorrelationInfoAccessor` implementation to retrieve this `CorrelationInfo` model, +or use the default `DefaultCorrelationInfoAccessor` implementation that stores this model + +**Example** +Name: `OperationId` +Value: `52EE2C00-53EE-476E-9DAB-C1234EB4AD0B` + +Name: `TransactionId` +Value: `0477E377-414D-47CD-8756-BCBE3DBE3ACB` + +**Usage** + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Serilog; + +IServiceProvider serviceProvider = ... +ILogger logger = new LoggerConfiguration() + .Enrich.WithCorrelationInfo(serviceProvider) + .CreateLogger(); + +logger.Information("This event will be enriched with the correlation information"); +``` + +Or alternatively, with a custom `ICorrelationInfoAccessor`: + +```csharp +using Arcus.Observability.Correlation; +using Serilog; + +ICorrelationInfoAccessor myCustomAccessor = ... + +ILogger logger = new LoggerConfiguration() + .Enrich.WithCorrelationInfo(myCustomAccessor) + .CreateLogger(); + +logger.Information("This event will be enriched with the correlation information"); +// Output: This event will be enriched with the correlation information {OperationId: 52EE2C00-53EE-476E-9DAB-C1234EB4AD0B, TransactionId: 0477E377-414D-47CD-8756-BCBE3DBE3ACB} +``` + +### Custom Serilog property names +The correlation information enricher allows you to specify the names of the log properties that will be added to the log event during enrichment. +This is available on all extension overloads. By default the operation ID is set to `OperationId` and the transaction ID to `TransactionId`. + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Serilog; + +IServiceProvider serviceProvider = ... +ILogger logger = new LoggerConfiguration() + .Enrich.WithCorrelationInfo( + serviceProvider, + operationIdPropertyName: "MyOperationId", + transactionIdPropertyName: "MyTransactionId") + .CreateLogger(); + +logger.Information("Some event"); +// Output: Some event {MyOperationId: 52EE2C00-53EE-476E-9DAB-C1234EB4AD0B, MyTransactionId: 0477E377-414D-47CD-8756-BCBE3DBE3ACB} +``` + +## Kubernetes Enricher +The `Arcus.Observability.Telemetry.Serilog.Enrichers` library provides a [Kubernetes](https://kubernetes.io/) [Serilog enricher](https://github.com/serilog/serilog/wiki/Enrichment) +that adds several machine information from the environment (variables). + +**Example** + +| Environment Variable | Log Property | +| ---------------------- | ------------ | +| `KUBERNETES_NODE_NAME` | NodeName | +| `KUBERNETES_POD_NAME` | PodName | +| `KUBERNETES_NAMESPACE` | Namespace | + +**Usage** + +```csharp +using Serilog; + +ILogger logger = new LoggerConfiguration() + .Enrich.WithKubernetesInfo() + .CreateLogger(); + +logger.Information("Some event"); +// Output: Some event {NodeName: demo-cluster, PodName: demo-app, Namespace: demo} +``` + +Here is an example of a Kubernetes YAML that provides the required environment variables: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: demo-app + labels: + app: demo +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: demo-app + app.kubernetes.io/instance: instance + template: + metadata: + labels: + app.kubernetes.io/name: demo-app + app.kubernetes.io/instance: instance + spec: + containers: + - name: event-proxy + image: arcusazure/arcus-event-grid-proxy + env: + - name: ARCUS_EVENTGRID_TOPICENDPOINT + value: https://arcus.io + - name: ARCUS_EVENTGRID_AUTHKEY + valueFrom: + secretKeyRef: + name: secrets-order-consumer + key: servicebus-connectionstring + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: KUBERNETES_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +``` + +### Custom Serilog property names +The Kubernetes enricher allows you to specify the names of the log properties that will be added to the log event during enrichment. +By default the node name is set to `NodeName`, the pod name to `PodName`, and the namespace to `Namespace`. + +```csharp +using Serilog; + +ILogger logger = new LoggerConfiguration() + .Enrich.WithKubernetesInfo( + nodeNamePropertyName: "MyNodeName", + podNamePropertyName: "MyPodName", + namespacePropertyName: "MyNamespace") + .CreateLogger(); + +logger.Information("Some event"); +// Output: Some event {MyNodeName: demo-cluster, MyPodName: demo-app, MyNamespace: demo} +``` + +## Version Enricher +The `Arcus.Observability.Telemetry.Serilog.Enrichers` library provides a [Serilog enricher](https://github.com/serilog/serilog/wiki/Enrichment) +that adds (by default) the current runtime assembly version of the product to the log event as a log property with the name `version`. + +**Example** +Name: `version` +Value: `1.0.0-preview` + +**Usage** + +```csharp +using Serilog; + +ILogger logger = new LoggerConfiguration() + .Enrich.WithVersion() + .CreateLogger(); + +logger.Information("Some event"); +// Output: Some event {version: 1.0.0-preview} +``` + +### Custom application version +The version enricher allows you to specify an `IAppVersion` instance that retrieves your custom application version, which will be used during enrichment. +By default this is set to the version of the current executing assembly. + +**Assembly version as application version** + +```csharp +using Microsoft.Extensions.DependencyInjection; + +public void ConfigureServices(IServiceCollection services) +{ + // Register the `AssemblyAppVersion` instance to retrieve the application version from the assembly where the passed-along `Startup` type is located. + services.AddAssemblyAppVersion(); +} +``` + +**User-provided version** + +```csharp +using Arcus.Observability.Telemetry.Core; +using Serilog; + +IAppVersion appVersion = new MyCustomAppVersion("v0.1.0"); +ILogger logger = new LoggerConfiguration() + .Enrich.WithVersion(appVersion) + .CreateLogger(); + +logger.Information("Some event"); +// Output: Some event {version: v0.1.0} +``` + +Or alternatively, you can choose to register the application version so you can use it in your application as well. + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +public void ConfigureServivces(IServiceCollection services) +{ + // Register the `MyApplicationVersion` instance to the registered services (using empty constructor). + services.AddAppVersion(); + + // Register the `MyApplicationVersion` instance using the service provider. + services.AddAppVersion(serviceProvider => + { + var logger = serviceProvider.GetRequiredService>(); + return new MyApplicationVersion(logger); + }); +} +``` + +Once the application version is registered, you can pass along the `IServiceProvider` instead to the Serilog configuration. + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Serilog; + +IServiceProvider serviceProvider = ... +ILogger logger = new LoggerConfiguration() + .Enrich.WithVersion(serviceProvider) + .CreateLogger(); +``` + +### Custom Serilog property names +The version enricher allows you to specify the name of the property that will be added to the log event during enrichment. +By default this is set to `version`. + +```csharp +using Serilog; + +ILogger logger = new LoggerConfiguration() + .Enrich.WithVersion(propertyName: "MyVersion") + .CreateLogger(); + +logger.Information("Some event"); +// Output: Some event {MyVersion: 1.0.0-preview} +``` \ No newline at end of file diff --git a/docs/versioned_docs/version-v2.7.0/03-Features/telemetry-filter.md b/docs/versioned_docs/version-v2.7.0/03-Features/telemetry-filter.md new file mode 100644 index 00000000..090613c6 --- /dev/null +++ b/docs/versioned_docs/version-v2.7.0/03-Features/telemetry-filter.md @@ -0,0 +1,54 @@ +--- +title: "Telemetry Filters" +layout: default +--- + +# Telemetry Filters + +## Installation + +This feature requires to install our NuGet package + +```shell +PM > Install-Package Arcus.Observability.Telemetry.Serilog.Filters +``` + +## Telemetry Type Filter + +In certain scenarios, the cost of having too much telemetry or having a certain type of telemetry in your logging can be too high to be useful. We could, for example, filter out in container logs the Request, Metrics and Dependencies, and discard all the other types to make the logging output more readable and cost-effective. + +This [Serilog filter](https://github.com/serilog/serilog/wiki/Enrichment) allows you to filter out different a specific type of telemetry. + +```csharp +using Arcus.Observability.Telemetry.Core; +using Arcus.Observability.Telemetry.Serilog.Filters; +using Serilog.Core; +using Serilog.Configuration; + +ILogger logger = new LoggerConfiguration() + .WriteTo.AzureApplicationInsightsWithConnectionString("") + .Filter.With(TelemetryTypeFilter.On(TelemetryType.Events)) + .CreateLogger(); +``` + +The filter can also be used to reduce telemetry for multiple types by chaining them: + +```csharp +ILogger logger = new LoggerConfiguration() + .WriteTo.AzureApplicationInsightsWithConnectionString("") + .Filter.With(TelemetryTypeFilter.On(TelemetryType.Events)) + .Filter.With(TelemetryTypeFilter.On(TelemetryType.Dependency)) + .CreateLogger(); +``` + +Alternatively, you can explicitly specify if it should track telemetry or not based on the application configuration has to be tracked or not: + +```csharp +var trackDependencies = configuration["telemetry:dependencies:isEnabled"]; +ILogger logger = new LoggerConfiguration() + .WriteTo.AzureApplicationInsightsWithConnectionString("") + .Filter.With(TelemetryTypeFilter.On(TelemetryType.Dependency, isTrackingEnabled: bool.Parse(trackDependencies))) + .CreateLogger(); +``` + + diff --git a/docs/versioned_docs/version-v2.7.0/03-Features/writing-different-telemetry-types.md b/docs/versioned_docs/version-v2.7.0/03-Features/writing-different-telemetry-types.md new file mode 100644 index 00000000..a62b76d3 --- /dev/null +++ b/docs/versioned_docs/version-v2.7.0/03-Features/writing-different-telemetry-types.md @@ -0,0 +1,697 @@ +--- +title: "Write different telemetry types" +layout: default +--- + +# Write different telemetry types + +Logs are a great way to gain insights, but sometimes they are not the best approach for the job. + +We provide the capability to track the following telemetry types on top of ILogger with good support on Serilog. +For most optimal output, we recommend using our [Azure Application Insights sink](./sinks/azure-application-insights.md). + +**We highly encourage to provide contextual information to all your telemetry** to make it more powerful and support this for all telemetry types. + +> :bulb: For sake of simplicity we have not included how to track contextual information, for more information see [our documentation](./making-telemetry-more-powerful.md). + +## Installation + +This feature requires to install our NuGet package + +```shell +PM > Install-Package Arcus.Observability.Telemetry.Core +``` + +## Dependencies + +Dependencies allow you to track how your external dependencies are doing to give you insights on performance and error rate. + +Since measuring dependencies can add some noise in your code, we've introduced `DependencyMeasurement` to make it simpler. ([docs](#making-it-easier-to-measure-dependencies)) +Linking service-to-service correlation can be hard, this can be made easier with including dependency ID's. ([docs](#making-it-easier-to-link-services)) + +### Measuring Azure Blob Storage dependencies + +We allow you to measure Azure Blob Storage dependencies. + +Here is how you can report a dependency call: + +```csharp +using Microsoft.Extensions.Logging; + +var durationMeasurement = new Stopwatch(); + +// Start measuring +durationMeasurement.Start(); +var startTime = DateTimeOffset.UtcNow; + +logger.LogBlobStorageDependency(accountName: "multimedia", containerName: "images", isSuccessful: true, startTime, durationMeasurement.Elapsed); +// Output: {"DependencyType": "Azure blob", "DependencyName": "images", "TargetName": "multimedia", "Duration": "00:00:00.2521801", "StartTime": "03/23/2020 09:56:31 +00:00", "IsSuccessful": true, "Context": {}} +``` + +### Measuring Azure Cosmos DB dependencies + +We allow you to measure Azure Cosmos dependencies. + +Here is how you can report a dependency call: + +**Cosmos SQL** + +```csharp +using Microsoft.Extensions.Logging; + +var durationMeasurement = new Stopwatch(); + +// Start measuring +durationMeasurement.Start(); +var startTime = DateTimeOffset.UtcNow; + +logger.LogCosmosSqlDependency(accountName: "administration", database: "docs", container: "purchases", isSuccessful: true, startTime: startTime, duration: durationMeasurement.Elapsed); +// Output: {"DependencyType": "Azure DocumentDB", "DependencyData": "docs/purchases", "TargetName": "administration", "Duration": "00:00:00.2521801", "StartTime": "03/23/2020 09:56:31 +00:00", "IsSuccessful" true, "Context": {}} +``` + +### Measuring Azure Event Hubs dependencies + +We allow you to measure Azure Event Hubs dependencies. + +Here is how you can report a dependency call: + +```csharp +using Microsoft.Extensions.Logging; + +var durationMeasurement = new Stopwatch(); + +// Start measuring +durationMeasurement.Start(); +var startTime = DateTimeOffset.UtcNow; + +logger.LogEventHubsDependency(namespaceName: "be.sensors.contoso", eventHubName: "temperature", isSuccessful: true, startTime: startTime, duration: durationMeasurement.Elapsed); +// Output: {"DependencyType": "Azure Event Hubs", "DependencyData": "be.sensors.contoso", "TargetName": "temperature", "Duration": "00:00:00.2521801", "StartTime": "03/23/2020 09:56:31 +00:00", "IsSuccessful": true, "Context": {}} +``` + +### Measuring Azure IoT Hub dependencies + +We allow you to measure Azure IoT Hub dependencies. + +**Example** + +Here is how you can report a dependency call: + +```csharp +using Microsoft.Extensions.Logging; + +var durationMeasurement = new Stopwatch(); + +// Start measuring +durationMeasurement.Start(); +var startTime = DateTimeOffset.UtcNow; + +logger.LogIotHubDependency(iotHubName: "sensors", isSuccessful: true, startTime: startTime, duration: durationMeasurement.Elapsed); +// Output: {"DependencyType": "Azure IoT Hub", "TargetName": "sensors", "Duration": "00:00:00.2521801", "StartTime": "03/23/2020 09:56:31 +00:00", "IsSuccessful": true, "Context": {}} +``` + +Or, alternatively you can pass along the IoT connection string itself so the host name will be selected for you. + +**Example** + +Here is how you can report a dependency call: + +```csharp +using Microsoft.Extensions.Logging; + +var durationMeasurement = new Stopwatch(); + +// Start measuring +durationMeasurement.Start(); +var startTime = DateTimeOffset.UtcNow; + +logger.LogIotHubDependencyWithConnectionString(iotHubConnectionString: "HostName=sensors.azure-devices.net;...", isSuccessful: true, startTime: startTime, duration: durationMeasurement.Elapsed); +// Output: {"DependencyType": "Azure IoT Hub", "TargetName": "sensors", "Duration": "00:00:00.2521801", "StartTime": "03/23/2020 09:56:31 +00:00", "IsSuccessful": true, "Context": {}} +``` + +> ⚠ Previously, an `LogIotHubDependency` IoT Hub connection string overload was available in the `Arcus.Observability.Telemetry.IoT` package but is now deprecated and renamed to `LogIotHubDependencyWithConnectionString` which is located in the `Arcus.Observability.Telemetry.Core` package. + +### Measuring Azure Key Vault dependencies + +We allow you to measure Azure Key vault dependencies. + +**Example** + +Here is how you can report a dependency call: + +```csharp +using Microsoft.Extensions.Logging; + +var durationMeasurement = new StopWatch(); + +// Start measuring +durationMeasurement.Start(); +var startTime = DateTimeOffset.UtcNow; + +logger.AzureKeyVaultDependency(vaultUri: "https://my-secret-store.vault.azure.net", secretName: "ServiceBus-ConnectionString", isSuccessful: true, startTime: startTime, duration: durationMeasurement.Elapsed); +// Output: {"DependencyType": "Azure key vault", "DependencyData": "ServiceBus-ConnectionString", "TargetName": "https://my-secret-store.vault.azure.net", "Duration": "00:00:00.2521801", "StartTime": "03/23/2020 09:56:31 +00:00", "IsSuccessful": true, "Context": {}} +``` + +### Measuring Azure Search dependencies + +We allow you to measure Azure Search dependencies for cognitive services. + +Here is how you can report an Azure Search dependency: + +```csharp +using Microsoft.Extensions.Logging; + +var durationMeasurement = new Stopwatch(); + +// Start measuring +durationMeasurement.Start(); +var startTime = DateTimeOffset.UtcNow; + +logger.LogAzureSearchDependency(searchServiceName: "orders-search", operationName: "get-orders", isSuccessful: true, startTime: startTime, duration: durationMeasurement.Elapsed); +// Output: {"DependencyType": "Azure Search", "DependencyData": "get-orders", "TargetName": "orders-search", "Duration": "00:00:00.2521801", "StartTime": "03/23/2020 09:56:31 +00:00", "IsSuccessful": true, "Context": {}} +``` + +### Measuring Azure Service Bus dependencies + +We allow you to measure Azure Service Bus dependencies for both queues & topics. + +Here is how you can report an Azure Service Bus Queue dependency: + +```csharp +using Microsoft.Extensions.Logging; + +var durationMeasurement = new Stopwatch(); + +// Start measuring +durationMeasurement.Start(); +var startTime = DateTimeOffset.UtcNow + +logger.LogServiceBusQueueDependency(queueName: "ordersqueue", isSuccessful: true, startTime: startTime, duration: durationMeasurement.Elapsed); +// Output: {"DependencyType": "Azure Service Bus", "TargetName": "ordersqueue", "Duration": "00:00:00.2521801", "StartTime": "03/23/2020 09:56:31 +00:00", "IsSuccessful": true, "Context": {"EntityType": "Queue"}} +``` + +Note that we have an `LogServiceBusTopicDependency` to log dependency logs for an Azure Service Bus Topic and an `LogServiceBusDependency` to log Azure Service Bus logs where the entity type is not known. + +### Measuring Azure Table Storage Dependencies + +We allow you to measure Azure Table Storage dependencies. + +Here is how you can report a dependency call: + +```csharp +using Microsoft.Extensions.Logging; + +var durationMeasurement = new Stopwatch(); + +// Start measuring +durationMeasurement.Start(); +var startTime = DateTimeOffset.UtcNow; + +logger.LogTableStorageDependency(accountName: "orderAccount", tableName: "orders", isSuccessful: true, startTime: startTime, duration: durationMeasurement.Elapsed); +// Output: {"DependencyType": "Azure table", "DependencyData": "orders", "TargetName": "orderAccount", "Duration": "00:00:00.2521801", "StartTime": "03/23/2020 09:56:31 +00:00", "IsSuccessful": true, "Context": {}} +``` + +### Measuring HTTP dependencies + +Here is how you can report a HTTP dependency: + +```csharp +using Microsoft.Extensions.Logging; + +var durationMeasurement = new Stopwatch(); + +// Create request +var request = new HttpRequestMessage(HttpMethod.Post, "http://requestbin.net/r/ujxglouj") +{ + Content = new StringContent("{\"message\":\"Hello World!\"") +}; + +// Start measuring +durationMeasurement.Start(); +var startTime = DateTimeOffset.UtcNow; +// Send request to dependant service +var response = await httpClient.SendAsync(request); + +logger.LogHttpDependency(request, statusCode: response.StatusCode, startTime: startTime, duration: durationMeasurement.Elapsed); +// Output: {"DependencyType" "Http", "DependencyName": "POST /r/ujxglouj", "TargetName": "requestbin.net", "ResultCode": 200, "Duration": "00:00:00.2521801", "StartTime": "03/23/2020 09:56:31 +00:00", "IsSuccessful": true, "Context": {}} +``` + +### Measuring SQL dependencies + +Here is how you can report a SQL dependency: + +```csharp +using Microsoft.Extensions.Logging; + +var durationMeasurement = new Stopwatch(); +string dependencyId = Guid.NewGuid().ToString(); + +// Start measuring +var startTime = DateTimeOffset.UtcNow; +durationMeasurement.Start(); + +// Interact with database +var products = await _repository.GetProducts(); + +logger.LogSqlDependency("Company SQL Server", "Stock Database", "GET ProductName FROM Products", "Get product names", isSuccessful: true, startTime: startTime, duration: durationMeasurement.Elapsed, dependencyId); +// Output: {"DependencyType": "Sql", "DependencyName": "Stock Database/Get product names", "DependencyData": "GET ProductName FROM Products", "TargetName": "Company SQL Server", "Duration": "00:00:01.2396312", "StartTime": "03/23/2020 09:32:02 +00:00", "IsSuccessful": true, "Context": {}} +``` + +Or alternatively, when one already got the SQL connection string, you can use the overload that takes this directly: + +```csharp +using Microsoft.Extensions.Logging; + +string connectionString = "Server=Company SQL Server;Database=Stock Database;User=admin;Password=123"; +string dependencyId = Guid.NewGuid().ToString(); +var durationMeasurement = new Stopwatch(); + +// Start measuring +var startTime = DateTimeOffset.UtcNow; +durationMeasurement.Start(); + +// Interact with database +var products = await _repository.GetProducts(); + +logger.LogSqlDependencyWithConnectionString(connectionString, "GET ProductName FROM Products", "Get product names", isSuccessful: true, measurement: measurement, dependencyId); +// Output: {"DependencyType": "Sql", "DependencyName": "Stock Database/Get product names", "DependencyData": "GET ProductName FROM Products", "TargetName": "Company SQL Server", "Duration": "00:00:01.2396312", "StartTime": "03/23/2020 09:32:02 +00:00", "IsSuccessful": true, "Context": {}} +``` + +> ⚠ This functionality was previously called `.LogSqlDependency` and was availabl in the `Arcus.Observability.Telemetry.Sql` package. This package is now deprecated. Use the `.LogSqlDependencyWithConnectionString` instead which is by-default available in the `Arcus.Observability.Telemetry.Core` package. + +### Measuring custom dependencies + +Here is how you can measure a custom dependency: + +```csharp +using Microsoft.Extensions.Logging; + +var durationMeasurement = new Stopwatch(); + +// Start measuring +var startTime = DateTimeOffset.UtcNow; +durationMeasurement.Start(); + +string dependencyName = "SendGrid"; +object dependencyData = "http://my.sendgrid.uri/" + +logger.LogDependency("SendGrid", dependencyData, isSuccessful: true, startTime: startTime, duration: durationMeasurement.Elapsed); +// Output: {"DependencyType": "SendGrid", "DependencyData": "http://my.sendgrid.uri/", "Duration": "00:00:01.2396312", "StartTime": "03/23/2020 09:32:02 +00:00", "IsSuccessful": true, "Context": {}} +``` + +### Making it easier to measure telemetry + +Measuring dependencies or requests means you need to keep track of how long the action took and when it started. +The `Arcus.Observability.Telemetry.Core` library provides an easy way to accomplish this. + +Here's a small example: + +```csharp +using Microsoft.Extensions.Logging; + +var durationMeasurement = new Stopwatch(); +var startTime = DateTimeOffset.UtcNow; +durationMeasurement.Start(); + +// Do action + +/// Track dependency +string dependencyName = "SendGrid"; +object dependencyData = "https://my.sendgrid.uri/"; +logger.LogDependency("SendGrid", dependencyData, isSuccessful: true, startTime: startTime, duration: durationMeasurement.Elapsed, context: telemetryContext); +``` + +#### Making it easier to measure dependencies + +By using `DurationMeasurement.Start()` we take care of the measuring aspect: + +```csharp +using Arcus.Observability.Telemetry.Core; +using Microsoft.Extensions.Logging; + +// Start measuring +using (var measurement = DurationMeasurement.Start()) +{ + // Do Action + + // Track dependency + string dependencyName = "SendGrid"; + object dependencyData = "https://my.sendgrid.uri/"; + logger.LogDependency(dependencyName, dependencyData, isSuccessful: true, measurement, telemetryContext); +} +``` + +Failures during the interaction with the dependency can be controlled by passing `isSuccessful`: + +```csharp +using Arcus.Observability.Telemetry.Core; +using Microsoft.Extensions.Logging; + +string dependencyName = "SendGrid"; +object dependencyData = "https://my.sendgrid.uri"; + +try +{ + // Interact with SendGrid... + // Done! + + logger.LogDependency(dependencyName, dependencyData, isSuccessful: true, measurement, telemetryContext); +} +catch (Exception exception) +{ + logger.LogError(exception, "Failed to interact with SendGrid"); + logger.LogDependency(dependencyName, dependencyData, isSuccessful: false, measurement, telemetryContext); +} +``` + +#### Making it easier to measure requests + +By using `DurationMeasurement.Start()` we take care of the measuring aspect: + +```csharp +using Arcus.Observability.Telemetry.Core; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; + +HttpRequest request = ... +HttpResponse response = ... + +// Start measuring +using (var measurement = DurationMeasurement.Start()) +{ + // Process message + + // Track request + logger.LogRequest(request, response, measurement, telemetryContext); +} +``` + +### Making it easier to link services + +Service-to-service correlation requires linkage between tracked dependencies (outgoing) and requests (incoming). +Tracking any kind of dependency with the library has the possibility to provide an dependency ID. + +To link the request (incoming) with the dependency (outgoing), the request needs to include this dependency ID in its tracking (dependency ID = request's parent ID) so that we now which dependency triggered the request. +For more information, see how to do this in a Web API and [Azure Service Bus](#incoming-azure-service-bus-requests) context. + +Tracking the outgoing dependency: + +```csharp +var durationMeasurement = new StopWatch(); + +// Start measuring +durationMeasurement.Start(); +var startTime = DateTimeOffset.UtcNow; + +var trackingId = "75D298F7-99FF-4BB8-8019-230974EB6D1E"; + +logger.AzureKeyVaultDependency( + vaultUri: "https://my-secret-store.vault.azure.net", + secretName: "ServiceBus-ConnectionString", + isSuccessful: true, + startTime: startTime, + duration: durationMeasurement.Elapsed, + dependencyId: trackingId); + +// Output: {"DependencyType": "Azure key vault", "DependencyId": "75D298F7-99FF-4BB8-8019-230974EB6D1E", "DependencyData": "ServiceBus-ConnectionString", "TargetName": "https://my-secret-store.vault.azure.net", "Duration": "00:00:00.2521801", "StartTime": "03/23/2020 09:56:31 +00:00", "IsSuccessful": true, "Context": {}} +``` + +## Events + +Events allow you to report custom events which are a great way to track business-related events. + +Here is how you can report an `Order Created` event: + +```csharp +using Microsoft.Extensions.Logging; + +logger.LogCustomEvent("Order Created"); +// Output: {"EventName": "Order Created", "Context": {}} +``` + +### Security Events + +Some events are considered "security events" when they relate to possible malicious activity, authentication, input validation... + +Here is how an invalid `Order` can be reported: + +```csharp +using Microsoft.Extensions.Logging; + +loger.LogSecurityEvent("Invalid Order"); +// Output: {"EventName": "Invalid Order", "Context": {"EventType": "Security"}} +``` + +## Metrics + +Metrics allow you to report custom metrics which allow you to give insights on application-specific metrics. + +Here is how you can report an `Invoices Received` metric: + +```csharp +using Microsoft.Extensions.Logging; + +logger.LogCustomMetric("Invoices Received", 133); +// Output: {"MetricName": "Invoices Received", "MetricValue": 133, "Timestamp": "03/23/2020 09:32:02 +00:00", "Context: {[TelemetryType, Metric]}} +``` + +## Requests + +### Incoming Azure Service Bus requests +Requests allow you to keep track of incoming Azure Service Bus messages on a queue or topic. + +Here is how you can log an Azure Service Bus queue request on a message that's being processed: + +```csharp +using Microsoft.Extensions.Logging; + +bool isSuccessful = false; + +// Start measuring. +using (var measurement = DurationMeasurement.Start()) +{ + try + { + // Processing message. + + // End processing. + + isSuccessful = true; + } + finally + { + logger.LogServiceBusQueueRequest(".servicebus.windows.net", "", "", isSuccessful, measurement); + // Output: Azure Service Bus from completed in 0.00:12:20.8290760 at 2021-10-26T05:36:03.6067975 +02:00 - (IsSuccessful: True, Context: {[ServiceBus-Endpoint, .servicebus.windows.net]; [ServiceBus-Entity, ]; [ServiceBus-EntityType, Queue]; [TelemetryType, Request]}) + } +} +``` + +💡 See [other possible domain names for Azure Service Bus](https://learn.microsoft.com/en-us/rest/api/servicebus/) to pass along as suffix with your Azure Service Bus namespace. + +We provide support for all Azure Service Bus entity types such as queues, topics and subscriptions. +All these types can be tracked by passing along the full Azure Service namespace, or with providing the namespace name and the Azure cloud separately. + +```csharp + +DependencyMeasurement measurement = ... + +// Tracking Azure Service Bus topics. +// ---------------------------------- + +// Providing the full Azure Service Bus topic namespace. +logger.LogServiceBusTopicRequest(".servicebus.windows.net", "", "", "", isSuccessful: true, measurement); + +// Providing the Azure Service Bus topic name and Azure cloud separately. +logger.LogServiceBusTopicRequestWithSuffix("", serviceBusNamespaceSuffix: ".servicebus.windows.net", "", "", "", isSuccessful: true, measurement); + + +// Tracking general Azure Service Bus requests. +// -------------------------------------------- + +// Providing the full Azure Service Bus topic namespace. +logger.LogServiceBusRequest(".servicebus.windows.net", "", "", "", isSuccessful: true, measurement, ServiceBusEntityType.Topic); + +// Providing the Azure Service Bus queue namespace name and Azure cloud separately. +logger.LogServiceBusQueueRequestWithSuffix("", serviceBusNamespaceSuffix: ".servicebus.windows.net", "", "", isSuccessful: true, measurement, ServiceBusEntityType.Queue); +``` + +### Incoming Azure EventHubs requests +Requests allow you to keep track of incoming Azure EventHubs event messages. + +Here is how you can log an Azure EventHubs request on an event that's being processed: + +```csharp +using Microsoft.Extensions.Logging; + +bool isSuccessful = false; + +// Start measuring. +using (var measurement = DurationMeasurement.Start()) +{ + try + { + // Processing message. + + // End processing. + + isSuccessful = true; + } + finally + { + logger.LogEventHubsRequest("", "", isSuccessful, measurement); + // Output: Azure EventHubs from Process completed in 0.00:12:20.8290760 at 2021-10-26T05:36:03.6067975 +02:00 - (IsSuccessful: True, Context: {[EventHubs-Namespace, ]; [EventHubs-Name, ]; [EventHubs-ConsumerGroup, $Default]; [TelemetryType, Request]}) + } +} +``` + +We provide overloads to configure the Azure EventHubs consumer group (default: `$Default`) and a functional operation name (default: `Process`). + +### Incoming HTTP requests in API's +Requests allow you to keep track of the HTTP requests that are performed against your API and what the response was that was sent out. + +**Installation** + +If you want to track the `HttpRequest` and `HttpResponse` of an ASP.NET Core project, you'll have to install an additional package to include these ASP.NET Core dependencies: + +```shell +PM > Install-Package Arcus.Observability.Telemetry.AspNetCore +``` + +**Example** + +Here is how you can keep track of requests: + +```csharp +using Microsoft.Extensions.Logging; + +// Determine calling tenant +string tenantName = "Unknown"; +if (httpContext.Request?.Headers?.ContainsKey("X-Tenant") == true) +{ + tenantName = httpContext.Request.Headers["X-Tenant"]; +} + +// Start tracking request. +using (var measurement = DurationMeasurement.Start()) +{ + // Perform action that creates a response, in this case call next middleware in the chain. + await _next(httpContext); + + logger.LogRequest(httpContext.Request, httpContext.Response, measurement); + // Output: {"RequestMethod": "GET", "RequestHost": "http://localhost:5000/", "RequestUri": "http://localhost:5000/weatherforecast", "ResponseStatusCode": 200, "RequestDuration": "00:00:00.0191554", "RequestTime": "03/23/2020 10:12:55 +00:00", "Context": {}} +} +``` + +> 💡 Note that [Arcus Web API request tracking middleware](https://webapi.arcus-azure.net/features/logging#logging-incoming-requests) can already do this for you in a ASP.NET Core application + +### Incoming HTTP requests in Azure Function HTTP trigger +Requests allow you to keep track of the HTTP requests that are performed against your Azure Function HTTP trigger and what the response was that was sent out. + +**Installation** + +For isolated Azure Functions, you'll have to install an additional package if you want to track the `HttpRequestData`: + +```shell +PM > Install-Package Arcus.Observability.Telemetry.AzureFunctions +``` + +**Example** + +Here is how you can keep track of requests: + +```csharp +using System.Net; +using Microsoft.Extensions.Logging; + +public async Task Run( + [HttpTrigger(AuthorizationLevel.Function, "post", Route = "v1/order")] HttpRequestData request, + FunctionContext executionContext) +{ + var statusCode = default(HttpStatusCode); + + // Start tracking request. + using (var measurement = DurationMeasurement.Start()) + { + try + { + // Processing... + + statusCode = HttpStatusCode.Ok; + return request.CreateResponse(statusCode) + } + finally + { + logger.LogRequest(request, statusCode, measurement); + // Output: {"RequestMethod": "POST", "RequestHost": "http://localhost:5000/", "RequestUri": "http://localhost:5000/v1/order", "ResponseStatusCode": 200, "RequestDuration": "00:00:00.0191554", "RequestTime": "03/23/2020 10:12:55 +00:00", "Context": {}} + } + } +} +``` + +### Incoming custom requests +Requests allow you to keep track of incoming messages. We provide an extension to track type of requests that aren't out-of-the-box so you can track your custom systems. + +Here is how you can log a custom request on an event that's being processed: + +```csharp +using Microsoft.Extensions.Logging; + +bool isSuccessful = false; + +// Start measuring. +using (var measurement = DurationMeasurement.Start()) +{ + try + { + // Processing message. + + // End processing. + + isSuccessful = true; + } + finally + { + logger.LogCustomRequest("", "", isSuccessful, measurement); + // Output: Custom from Process completed in 0.00:12:20.8290760 at 2021-10-26T05:36:03.6067975 +02:00 - (IsSuccessful: True, Context: {[TelemetryType, Request]}) + } +} +``` + +The `` will reflect the `Source` in Application Insights telemetry. This is set automatically in our HTTP, Azure Service Bus, Azure EventHubs, etc. requests but is configurable when you track custom requests. +We provide overloads to configure the functional operation name (default: `Process`). + +## Traces & Exceptions +Application Insights telemetry traces and exceptions are log messages not directly linked by an incoming request, outgoing dependency, or metric. +These traces are also linked with correlation and are therefore part of the whole application component in Application Insights. + +Traces and exceptions can be logged with the general Microsoft logging extensions like: `LogInformation`, `LogWarning`, `LogError`... +To help with logging useful traces, we provide several overloads on these existing extensions to pass in additional contextual information. + +> ⚠ Note that log messages that gets an exception passed-in, are tracked as exceptions in Application Insights, and not as traces. + +```csharp +using Microsoft.Extensions.Logging; + +var telemetryContext = new Dictionary +{ + ["Order_Id"] = "abc-def", + ["Order_Date"] = DateTimeOffset.UtcNow +}; + +ILogger logger = ... + +// Informational messages (traces) +logger.LogInformation("This is an informational message!", telemetryContext); +// > Result in trace with telemetry context. + +logger.LogInformation("this is an informational message with an {Argument}!", telemetryContext, "The argument"); +// > Result in trace with telemetry context. + + +// Error messages (traces / exceptions) +logger.LogError("This is an error message!", telemetryContext); +// > Result in trace with telemetry context. + +var exception = new ApplicationException("Something happened in the application!"); +logger.LogError(exception, "This is an error message!", telemetryContext); +// > Result in exception with telemetry context. +``` \ No newline at end of file diff --git a/docs/versioned_sidebars/version-v2.7.0-sidebars.json b/docs/versioned_sidebars/version-v2.7.0-sidebars.json new file mode 100644 index 00000000..992947bd --- /dev/null +++ b/docs/versioned_sidebars/version-v2.7.0-sidebars.json @@ -0,0 +1,8 @@ +{ + "version-v2.7.0/tutorialSidebar": [ + { + "type": "autogenerated", + "dirName": "." + } + ] +} diff --git a/docs/versions.json b/docs/versions.json index e6b4c5a4..ebe3013b 100644 --- a/docs/versions.json +++ b/docs/versions.json @@ -1,4 +1,5 @@ [ + "v2.7.0", "v2.6.0", "v2.5.0", "v2.4",