prep for centralized logging with serilog and elasticsearch

skills from two courses were utilized here: - Implementing Cross-cutting Concerns for ASP.NET Core 3 Microservices by Steve Gordan - Logging and Monitoring in ASP.NET Core 6 by Erik Dahl since this is a prep, elasticsearch is not enabled by default once the solution is containerized and orchestrated locally, then elasticsearch and kibana will come into play as containers i've enriched the logs with the informational version from the assembly and custom metadata that may or may not be added to overtime currently, a link to the commit in github is an extra piece of metadata in the assembly which will be useful in the logs to investigate exceptions some az-400 skills also come into play here, notably: - develop an instrumentation strategy - assess and configure a logging framework - design a log aggregation and storage strategy - design a log aggregation and query strategy - design and implement distributed tracing for distributed tracing, aspnetcore sets the traceId when requests are made. in kibana or azure log analytics, querying by the traceId will provide all the logs relevant for a request if it fails and one can determine where the system failed to handle the request and why those objectives mentioned earlier are from the az-400 that i studied for and took, the exam looks very different now another note about those objectives, those objectives are done in code here. in the exam, for example, distributed tracing was tested using the application map of application insights and ofcourse, the projects have to be instrumented with application insights. i didn't do that here, not too sure how possible that is when the solution is in azure kubernetes services but i'll figure that out when i get there in azure kubernetes services, the monitoring addon will be enabled [also from the exam]
ShaylenReddy42 · Sep 25, 2022 · 08bd63d · 08bd63d
1 parent 0e6be05
commit 08bd63d
Show file tree

Hide file tree

Showing 26 changed files with 340 additions and 31 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -60,6 +60,7 @@ endif()
 
 set(FILE_VERSION "${TAG}.${REVISION}")
 set(INFORMATIONAL_VERSION "${TAG}+${REVISION}-${COMMIT_ID}")
+set(COMMIT_URL "https://github.com/ShaylenReddy42/Seelans-Tyres/commit/${COMMIT_ID}")
 
 message(STATUS "File Version: ${FILE_VERSION}")
 message(STATUS "Informational Version: ${INFORMATIONAL_VERSION}")

diff --git a/Frontends/SeelansTyres.Frontends.Mvc/Program.cs b/Frontends/SeelansTyres.Frontends.Mvc/Program.cs
@@ -6,9 +6,47 @@
 using Microsoft.AspNetCore.Authentication.OpenIdConnect;
 using Microsoft.AspNetCore.Authentication;
 using System.Security.Claims;
+using Serilog;
+using Serilog.Exceptions;
+using Serilog.Sinks.Elasticsearch;
+using Serilog.Events;
+using System.Reflection;
 
 var builder = WebApplication.CreateBuilder(args);
 
+builder.Logging.ClearProviders();
+
+var assembly = typeof(Program).Assembly;
+
+builder.Host.UseSerilog((hostBuilderContext, loggerConfiguration) =>
+{
+    loggerConfiguration
+        .ReadFrom.Configuration(hostBuilderContext.Configuration)
+        .Enrich.FromLogContext()
+        .Enrich.WithExceptionDetails()
+        .Enrich.WithProperty("Application Name", hostBuilderContext.HostingEnvironment.ApplicationName)
+        .Enrich.WithProperty("Descriptive Application Name", assembly.GetCustomAttribute<AssemblyProductAttribute>()!.Product)
+        .Enrich.WithProperty("Codebase Version", $"v{assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()!.InformationalVersion}")
+        .WriteTo.Console();
+
+    var metadata = assembly.GetCustomAttributes<AssemblyMetadataAttribute>().ToList();
+
+    metadata.ForEach(attribute => loggerConfiguration.Enrich.WithProperty(attribute.Key, attribute.Value));
+
+    if (hostBuilderContext.Configuration.GetValue<bool>("LoggingSinks:Elasticsearch:Enabled") is true)
+    {
+        loggerConfiguration
+            .WriteTo.Elasticsearch(
+                new ElasticsearchSinkOptions(new Uri(hostBuilderContext.Configuration["LoggingSinks:Elasticsearch:Url"]))
+                {
+                    AutoRegisterTemplate = true,
+                    AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7,
+                    IndexFormat = "seelanstyres-logs-{0:yyyy.MM.dd}",
+                    MinimumLogEventLevel = LogEventLevel.Debug
+                });
+    }
+});
+
 // Add services to the container.
 builder.Services.AddControllersWithViews();
 

diff --git a/Frontends/SeelansTyres.Frontends.Mvc/SeelansTyres.Frontends.Mvc.csproj b/Frontends/SeelansTyres.Frontends.Mvc/SeelansTyres.Frontends.Mvc.csproj
@@ -40,6 +40,10 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
+    <PackageReference Include="Serilog.AspNetCore" Version="6.0.1" />
+    <PackageReference Include="Serilog.Exceptions" Version="8.4.0" />
+    <PackageReference Include="Serilog.Settings.Configuration" Version="3.4.0" />
+    <PackageReference Include="Serilog.Sinks.Elasticsearch" Version="8.4.1" />
   </ItemGroup>
 
 </Project>
diff --git a/Frontends/SeelansTyres.Frontends.Mvc/appsettings.json b/Frontends/SeelansTyres.Frontends.Mvc/appsettings.json
@@ -1,8 +1,16 @@
 {
-  "Logging": {
-    "LogLevel": {
+  "Serilog": {
+    "MinimumLevel": {
       "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
+      "Override": {
+        "Microsoft.AspNetCore": "Warning"
+      }
+    }
+  },
+  "LoggingSinks": {
+    "Elasticsearch": {
+      "Url": "undecided",
+      "Enabled": false
     }
   },
   "AllowedHosts": "*",

diff --git a/Gateways/SeelansTyres.Gateways.MvcBff/Program.cs b/Gateways/SeelansTyres.Gateways.MvcBff/Program.cs
@@ -4,12 +4,50 @@
 using Ocelot.Middleware;
 using SeelansTyres.Gateways.MvcBff.DelegatingHandlers;
 using SeelansTyres.Gateways.MvcBff.Services;
+using Serilog;
+using Serilog.Events;
+using Serilog.Exceptions;
+using Serilog.Sinks.Elasticsearch;
+using System.Reflection;
 using static System.Net.Mime.MediaTypeNames;
 
 var builder = WebApplication.CreateBuilder(args);
 
 builder.Configuration.EnableSubstitutionsWithDelimitedFallbackDefaults("$(", ")", " ?? ");
 
+builder.Logging.ClearProviders();
+
+var assembly = typeof(Program).Assembly;
+
+builder.Host.UseSerilog((hostBuilderContext, loggerConfiguration) =>
+{
+    loggerConfiguration
+        .ReadFrom.Configuration(hostBuilderContext.Configuration)
+        .Enrich.FromLogContext()
+        .Enrich.WithExceptionDetails()
+        .Enrich.WithProperty("Application Name", hostBuilderContext.HostingEnvironment.ApplicationName)
+        .Enrich.WithProperty("Descriptive Application Name", assembly.GetCustomAttribute<AssemblyProductAttribute>()!.Product)
+        .Enrich.WithProperty("Codebase Version", $"v{assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()!.InformationalVersion}")
+        .WriteTo.Console();
+
+    var metadata = assembly.GetCustomAttributes<AssemblyMetadataAttribute>().ToList();
+
+    metadata.ForEach(attribute => loggerConfiguration.Enrich.WithProperty(attribute.Key, attribute.Value));
+
+    if (hostBuilderContext.Configuration.GetValue<bool>("LoggingSinks:Elasticsearch:Enabled") is true)
+    {
+        loggerConfiguration
+            .WriteTo.Elasticsearch(
+                new ElasticsearchSinkOptions(new Uri(hostBuilderContext.Configuration["LoggingSinks:Elasticsearch:Url"]))
+                {
+                    AutoRegisterTemplate = true,
+                    AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7,
+                    IndexFormat = "seelanstyres-logs-{0:yyyy.MM.dd}",
+                    MinimumLogEventLevel = LogEventLevel.Debug
+                });
+    }
+});
+
 var authenticationScheme = "SeelansTyresMvcBffAuthenticationScheme";
 
 builder.Services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme)

diff --git a/Gateways/SeelansTyres.Gateways.MvcBff/SeelansTyres.Gateways.MvcBff.csproj b/Gateways/SeelansTyres.Gateways.MvcBff/SeelansTyres.Gateways.MvcBff.csproj
@@ -22,6 +22,10 @@
     <PackageReference Include="IdentityModel" Version="6.0.0" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.9" />
     <PackageReference Include="Ocelot" Version="18.0.0" />
+    <PackageReference Include="Serilog.AspNetCore" Version="6.0.1" />
+    <PackageReference Include="Serilog.Exceptions" Version="8.4.0" />
+    <PackageReference Include="Serilog.Settings.Configuration" Version="3.4.0" />
+    <PackageReference Include="Serilog.Sinks.Elasticsearch" Version="8.4.1" />
   </ItemGroup>
 
 </Project>
diff --git a/Gateways/SeelansTyres.Gateways.MvcBff/appsettings.json b/Gateways/SeelansTyres.Gateways.MvcBff/appsettings.json
@@ -1,8 +1,16 @@
 {
-  "Logging": {
-    "LogLevel": {
+  "Serilog": {
+    "MinimumLevel": {
       "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
+      "Override": {
+        "Microsoft.AspNetCore": "Warning"
+      }
+    }
+  },
+  "LoggingSinks": {
+    "Elasticsearch": {
+      "Url": "undecided",
+      "Enabled": false
     }
   },
   "AllowedHosts": "*",

diff --git a/Services/SeelansTyres.Services.AddressService/Program.cs b/Services/SeelansTyres.Services.AddressService/Program.cs
@@ -6,10 +6,47 @@
 using SeelansTyres.Services.AddressService.Authorization;
 using SeelansTyres.Services.AddressService.Data;
 using SeelansTyres.Services.AddressService.Services;
+using Serilog;
+using Serilog.Events;
+using Serilog.Exceptions;
+using Serilog.Sinks.Elasticsearch;
 using System.Reflection;
 
 var builder = WebApplication.CreateBuilder(args);
 
+builder.Logging.ClearProviders();
+
+var assembly = typeof(Program).Assembly;
+
+builder.Host.UseSerilog((hostBuilderContext, loggerConfiguration) =>
+{
+    loggerConfiguration
+        .ReadFrom.Configuration(hostBuilderContext.Configuration)
+        .Enrich.FromLogContext()
+        .Enrich.WithExceptionDetails()
+        .Enrich.WithProperty("Application Name", hostBuilderContext.HostingEnvironment.ApplicationName)
+        .Enrich.WithProperty("Descriptive Application Name", assembly.GetCustomAttribute<AssemblyProductAttribute>()!.Product)
+        .Enrich.WithProperty("Codebase Version", $"v{assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()!.InformationalVersion}")
+        .WriteTo.Console();
+
+    var metadata = assembly.GetCustomAttributes<AssemblyMetadataAttribute>().ToList();
+
+    metadata.ForEach(attribute => loggerConfiguration.Enrich.WithProperty(attribute.Key, attribute.Value));
+
+    if (hostBuilderContext.Configuration.GetValue<bool>("LoggingSinks:Elasticsearch:Enabled") is true)
+    {
+        loggerConfiguration
+            .WriteTo.Elasticsearch(
+                new ElasticsearchSinkOptions(new Uri(hostBuilderContext.Configuration["LoggingSinks:Elasticsearch:Url"]))
+                {
+                    AutoRegisterTemplate = true,
+                    AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7,
+                    IndexFormat = "seelanstyres-logs-{0:yyyy.MM.dd}",
+                    MinimumLogEventLevel = LogEventLevel.Debug
+                });
+    }
+});
+
 // Add services to the container.
 
 builder.Services.AddControllers();

diff --git a/Services/SeelansTyres.Services.AddressService/SeelansTyres.Services.AddressService.csproj b/Services/SeelansTyres.Services.AddressService/SeelansTyres.Services.AddressService.csproj
@@ -26,6 +26,10 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="6.0.8" />
+    <PackageReference Include="Serilog.AspNetCore" Version="6.0.1" />
+    <PackageReference Include="Serilog.Exceptions" Version="8.4.0" />
+    <PackageReference Include="Serilog.Settings.Configuration" Version="3.4.0" />
+    <PackageReference Include="Serilog.Sinks.Elasticsearch" Version="8.4.1" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
   </ItemGroup>
 

diff --git a/Services/SeelansTyres.Services.AddressService/appsettings.json b/Services/SeelansTyres.Services.AddressService/appsettings.json
@@ -1,8 +1,16 @@
 {
-  "Logging": {
-    "LogLevel": {
+  "Serilog": {
+    "MinimumLevel": {
       "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
+      "Override": {
+        "Microsoft.AspNetCore": "Warning"
+      }
+    }
+  },
+  "LoggingSinks": {
+    "Elasticsearch": {
+      "Url": "undecided",
+      "Enabled": false
     }
   },
   "AllowedHosts": "*",

diff --git a/Services/SeelansTyres.Services.IdentityService/Program.cs b/Services/SeelansTyres.Services.IdentityService/Program.cs
@@ -12,26 +12,46 @@
 using Serilog;
 using Serilog.Events;
 using Serilog.Exceptions;
+using Serilog.Sinks.Elasticsearch;
 using Serilog.Sinks.SystemConsole.Themes;
 using System.Reflection;
 
 var builder = WebApplication.CreateBuilder(args);
 
+builder.Logging.ClearProviders();
+
+var assembly = typeof(Program).Assembly;
+
 builder.Host.UseSerilog((hostBuilderContext, loggerConfiguration) =>
 {
     loggerConfiguration
-        .MinimumLevel.Debug()
-        .MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
-        .MinimumLevel.Override("Microsoft.Hosting.Lifetime", LogEventLevel.Information)
-        .MinimumLevel.Override("System", LogEventLevel.Warning)
-        .MinimumLevel.Override("Microsoft.AspNetCore.Authentication", LogEventLevel.Information)
+        .ReadFrom.Configuration(hostBuilderContext.Configuration)
         .Enrich.FromLogContext()
         .Enrich.WithExceptionDetails()
-        .WriteTo.Console(outputTemplate: "[{Timestamp:HH:mm:ss} {Level}] {SourceContext}{NewLine}{Message:lj}{NewLine}{Exception}{NewLine}", theme: AnsiConsoleTheme.Code);
-});
+        .Enrich.WithProperty("Application Name", hostBuilderContext.HostingEnvironment.ApplicationName)
+        .Enrich.WithProperty("Descriptive Application Name", assembly.GetCustomAttribute<AssemblyProductAttribute>()!.Product)
+        .Enrich.WithProperty("Codebase Version", $"v{assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()!.InformationalVersion}")
+        .WriteTo.Console(
+            outputTemplate: "[{Timestamp:HH:mm:ss} {Level}] {SourceContext}{NewLine}{Message:lj}{NewLine}{Exception}{NewLine}", 
+            theme: AnsiConsoleTheme.Code);
 
-var connectionString = builder.Configuration["SeelansTyresIdentityContext"];
-var assemblyName = typeof(Program).Assembly.GetName().Name;
+    var metadata = assembly.GetCustomAttributes<AssemblyMetadataAttribute>().ToList();
+
+    metadata.ForEach(attribute => loggerConfiguration.Enrich.WithProperty(attribute.Key, attribute.Value));
+
+    if (hostBuilderContext.Configuration.GetValue<bool>("LoggingSinks:Elasticsearch:Enabled") is true)
+    {
+        loggerConfiguration
+            .WriteTo.Elasticsearch(
+                new ElasticsearchSinkOptions(new Uri(hostBuilderContext.Configuration["LoggingSinks:Elasticsearch:Url"]))
+                {
+                    AutoRegisterTemplate = true,
+                    AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7,
+                    IndexFormat = "seelanstyres-logs-{0:yyyy.MM.dd}",
+                    MinimumLogEventLevel = LogEventLevel.Debug
+                });
+    }
+});
 
 builder.Services.AddControllersWithViews();
 
@@ -59,6 +79,9 @@
     }});
 });
 
+var connectionString = builder.Configuration["SeelansTyresIdentityContext"];
+var assemblyName = assembly.GetName().Name;
+
 builder.Services.AddDbContext<CustomerContext>(options =>
 {
     options.UseSqlServer(connectionString, options => options.MigrationsAssembly(assemblyName));

diff --git a/Services/SeelansTyres.Services.IdentityService/SeelansTyres.Services.IdentityService.csproj b/Services/SeelansTyres.Services.IdentityService/SeelansTyres.Services.IdentityService.csproj
@@ -31,6 +31,8 @@
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="6.0.8" />
     <PackageReference Include="Serilog.AspNetCore" Version="6.0.1" />
     <PackageReference Include="Serilog.Exceptions" Version="8.4.0" />
+    <PackageReference Include="Serilog.Settings.Configuration" Version="3.4.0" />
+    <PackageReference Include="Serilog.Sinks.Elasticsearch" Version="8.4.1" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
   </ItemGroup>
 

diff --git a/Services/SeelansTyres.Services.IdentityService/Services/ConfigurationDataSeeder.cs b/Services/SeelansTyres.Services.IdentityService/Services/ConfigurationDataSeeder.cs
@@ -1,24 +1,25 @@
 using IdentityServer4.EntityFramework.DbContexts;
 using IdentityServer4.EntityFramework.Mappers;
-using Serilog;
 
 namespace SeelansTyres.Services.IdentityService.Services;
 
 public class ConfigurationDataSeeder
 {
 	private readonly ConfigurationDbContext context;
     private readonly IConfiguration configuration;
+    private readonly ILogger<ConfigurationDataSeeder> logger;
 
     public ConfigurationDataSeeder(
         ConfigurationDbContext context,
-        IConfiguration configuration) =>
-		    (this.context, this.configuration) = (context, configuration);
+        IConfiguration configuration,
+        ILogger<ConfigurationDataSeeder> logger) =>
+		    (this.context, this.configuration, this.logger) = (context, configuration, logger);
 
     public async Task SeedConfigurationDataAsync()
     {
         Config.Configuration = configuration;
 
-        Log.Debug("Clients being populated");
+        logger.LogDebug("Clients being populated");
         Config.Clients.ToList().ForEach(client =>
         {
             var clientEntity =
@@ -32,7 +33,7 @@ public async Task SeedConfigurationDataAsync()
         });
         await context.SaveChangesAsync();
 
-        Log.Debug("IdentityResources being populated");
+        logger.LogDebug("IdentityResources being populated");
         Config.IdentityResources.ToList().ForEach(identityResource =>
         {
             var identityResourceEntity =
@@ -46,7 +47,7 @@ public async Task SeedConfigurationDataAsync()
         });
         await context.SaveChangesAsync();
 
-        Log.Debug("ApiScopes being populated");
+        logger.LogDebug("ApiScopes being populated");
         Config.ApiScopes.ToList().ForEach(apiScope =>
         {
             var apiScopeEntity =
@@ -60,7 +61,7 @@ public async Task SeedConfigurationDataAsync()
         });
         await context.SaveChangesAsync();
 
-        Log.Debug("ApiResources being populated");
+        logger.LogDebug("ApiResources being populated");
         Config.ApiResources.ToList().ForEach(apiResource =>
         {
             var apiResourceEntity =