Skip to content

Commit

Permalink
Updates health check json response with additional descriptions and "…
Browse files Browse the repository at this point in the history
…data" to describe what makes the system degraded or unhealthy.
  • Loading branch information
Kritner committed Dec 4, 2020
1 parent eb49863 commit 952166e
Show file tree
Hide file tree
Showing 6 changed files with 77 additions and 59 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,11 @@ namespace Kritner.Orleans.GettingStarted.Grains.HealthChecks
[StatelessWorker(1)]
public class BasicHealthCheckGrain : Grain, IBasicHealthCheckGrain
{
private const string HealthCheckDescription = "A basic health check. Should only ever return healthy assuming the Orleans cluster is up.";

public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = new CancellationToken())
{
return Task.FromResult(new HealthCheckResult(HealthStatus.Healthy));
return Task.FromResult(new HealthCheckResult(HealthStatus.Healthy, HealthCheckDescription));
}
}
}
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Threading;
using System.Threading.Tasks;
using Kritner.Orleans.GettingStarted.GrainInterfaces.HealthChecks;
Expand All @@ -13,6 +15,12 @@ public class CpuHealthCheckGrain : Grain, ICpuHealthCheckGrain
{
private const float UnhealthyThreshold = 90;
private const float DegradedThreshold = 70;
private readonly ReadOnlyDictionary<string, object> HealthCheckData = new ReadOnlyDictionary<string, object>(
new Dictionary<string, object>()
{
{ "Unhealthy Threshold", UnhealthyThreshold},
{ "Degraded Threshold", DegradedThreshold}
});

private readonly IHostEnvironmentStatistics _hostEnvironmentStatistics;

Expand All @@ -25,23 +33,23 @@ public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, Canc
{
if (_hostEnvironmentStatistics.CpuUsage == null)
{
return Task.FromResult(HealthCheckResult.Unhealthy("Could not determine CPU usage."));
return Task.FromResult(HealthCheckResult.Unhealthy("Could not determine CPU usage.", data: HealthCheckData));
}

if (_hostEnvironmentStatistics.CpuUsage > UnhealthyThreshold)
{
return Task.FromResult(HealthCheckResult.Unhealthy(
$"CPU utilization is unhealthy at {_hostEnvironmentStatistics.CpuUsage}%."));
$"CPU utilization is unhealthy at {_hostEnvironmentStatistics.CpuUsage:0.00}%.", data: HealthCheckData));
}

if (_hostEnvironmentStatistics.CpuUsage > DegradedThreshold)
{
return Task.FromResult(HealthCheckResult.Degraded(
$"CPU utilization is degraded at {_hostEnvironmentStatistics.CpuUsage}%."));
$"CPU utilization is degraded at {_hostEnvironmentStatistics.CpuUsage:0.00}%.", data: HealthCheckData));
}

return Task.FromResult(HealthCheckResult.Healthy(
$"CPU utilization is healthy at {_hostEnvironmentStatistics.CpuUsage}%."));
$"CPU utilization is healthy at {_hostEnvironmentStatistics.CpuUsage:0.00}%.", data: HealthCheckData));
}
}
}
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Threading;
using System.Threading.Tasks;
using Kritner.Orleans.GettingStarted.GrainInterfaces.HealthChecks;
Expand All @@ -12,8 +14,15 @@ namespace Kritner.Orleans.GettingStarted.Grains.HealthChecks
public class MemoryHealthCheckGrain : Grain, IMemoryHealthCheckGrain
{
private const float UnhealthyThreshold = 95;
private const float DegradedThreshold = 90;

private const float DegradedThreshold = 90;

private readonly ReadOnlyDictionary<string, object> HealthCheckData = new ReadOnlyDictionary<string, object>(
new Dictionary<string, object>()
{
{ "Unhealthy Threshold", UnhealthyThreshold},
{ "Degraded Threshold", DegradedThreshold}
});

private readonly IHostEnvironmentStatistics _hostEnvironmentStatistics;

public MemoryHealthCheckGrain(IHostEnvironmentStatistics hostEnvironmentStatistics)
Expand All @@ -25,31 +34,31 @@ public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, Canc
{
if (_hostEnvironmentStatistics?.AvailableMemory == null || _hostEnvironmentStatistics?.TotalPhysicalMemory == null)
{
return Task.FromResult(HealthCheckResult.Unhealthy("Could not determine memory calculation."));
return Task.FromResult(HealthCheckResult.Unhealthy("Could not determine memory calculation.", data: HealthCheckData));
}

if (_hostEnvironmentStatistics?.AvailableMemory == 0 && _hostEnvironmentStatistics?.AvailableMemory == 0)
{
return Task.FromResult(HealthCheckResult.Unhealthy("Could not determine memory calculation."));
return Task.FromResult(HealthCheckResult.Unhealthy("Could not determine memory calculation.", data: HealthCheckData));
}

var memoryUsed = 100 - ((float)_hostEnvironmentStatistics.AvailableMemory / (float)_hostEnvironmentStatistics.TotalPhysicalMemory * 100);

if (memoryUsed > UnhealthyThreshold)
{
return Task.FromResult(HealthCheckResult.Unhealthy(
$"Memory utilization is unhealthy at {memoryUsed:0.00}%."));
$"Memory utilization is unhealthy at {memoryUsed:0.00}%.", data: HealthCheckData));

}

if (memoryUsed > DegradedThreshold)
{
return Task.FromResult(HealthCheckResult.Degraded(
$"Memory utilization is degraded at {memoryUsed:0.00}%."));
$"Memory utilization is degraded at {memoryUsed:0.00}%.", data: HealthCheckData));
}

return Task.FromResult(HealthCheckResult.Healthy(
$"Memory utilization is healthy at {memoryUsed:0.00}%."));
$"Memory utilization is healthy at {memoryUsed:0.00}%.", data: HealthCheckData));
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;

namespace Kritner.OrleansGettingStarted.SiloHost.Helpers
{
internal static class HealthCheckResponseWriter
{
public static Task WriteResponse(HttpContext context, HealthReport healthReport)
{
context.Response.ContentType = "application/json; charset=utf-8";

var result = JsonConvert.SerializeObject(new
{
status = healthReport.Status.ToString(),
details = healthReport.Entries.Select(e => new
{
key = e.Key,
description = e.Value.Description,
status = e.Value.Status.ToString(),
data = e.Value.Data
})
}, Formatting.Indented);

return context.Response.WriteAsync(result);
}
}
}
9 changes: 8 additions & 1 deletion src/Kritner.OrleansGettingStarted.SiloHost/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,14 @@ private static IHostBuilder CreateHostBuilder(string[] args, string env, IConfig
builder.AddConfiguration(configurationRoot);
})
.ConfigureServices(DependencyInjectionHelper.IocContainerRegistration)
.ConfigureWebHostDefaults(builder => { builder.UseStartup<Startup>(); })
.ConfigureLogging(logging => {
logging.AddConsole();
logging.SetMinimumLevel(LogLevel.Information);
})
.ConfigureWebHostDefaults(builder =>
{
builder.UseStartup<Startup>();
})
.UseOrleans(siloBuilder =>
{
siloBuilder
Expand Down
49 changes: 3 additions & 46 deletions src/Kritner.OrleansGettingStarted.SiloHost/Startup.cs
Original file line number Diff line number Diff line change
@@ -1,16 +1,11 @@
using System.IO;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using Kritner.OrleansGettingStarted.SiloHost.HealthChecks;
using Kritner.OrleansGettingStarted.SiloHost.Helpers;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Diagnostics.HealthChecks;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Hosting;

namespace Kritner.OrleansGettingStarted.SiloHost
Expand Down Expand Up @@ -43,6 +38,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
}

app.UseHttpsRedirection();
app.UseHsts();

app.UseRouting();

Expand All @@ -55,51 +51,12 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
new HealthCheckOptions
{
AllowCachingResponses = false,
ResponseWriter = WriteResponse
ResponseWriter = HealthCheckResponseWriter.WriteResponse
})
.WithMetadata(new AllowAnonymousAttribute());
endpoints.MapControllers();
});
}

private static Task WriteResponse(HttpContext context, HealthReport result)
{
context.Response.ContentType = "application/json; charset=utf-8";

var options = new JsonWriterOptions
{
Indented = true
};

using var stream = new MemoryStream();
using (var writer = new Utf8JsonWriter(stream, options))
{
writer.WriteStartObject();
writer.WriteString("status", result.Status.ToString());
writer.WriteStartObject("results");
foreach (var entry in result.Entries)
{
writer.WriteStartObject(entry.Key);
writer.WriteString("status", entry.Value.Status.ToString());
writer.WriteString("description", entry.Value.Description);
writer.WriteStartObject("data");
foreach (var item in entry.Value.Data)
{
writer.WritePropertyName(item.Key);
JsonSerializer.Serialize(
writer, item.Value, item.Value?.GetType() ??
typeof(object));
}
writer.WriteEndObject();
writer.WriteEndObject();
}
writer.WriteEndObject();
writer.WriteEndObject();
}

var json = Encoding.UTF8.GetString(stream.ToArray());

return context.Response.WriteAsync(json);
}
}
}

0 comments on commit 952166e

Please sign in to comment.