From 482cd8669be960113b444f11a464b365a1047ccb Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Fri, 7 Mar 2025 19:53:42 +1300 Subject: [PATCH 1/2] Expose metrics for health check failures. --- config/metrics.rb | 8 ++++++++ config/sus.rb | 11 +++++++++++ lib/async/container/generic.rb | 11 ++++++++--- lib/metrics/provider/async/container.rb | 6 ++++++ lib/metrics/provider/async/container/generic.rb | 17 +++++++++++++++++ 5 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 config/metrics.rb create mode 100644 lib/metrics/provider/async/container.rb create mode 100644 lib/metrics/provider/async/container/generic.rb diff --git a/config/metrics.rb b/config/metrics.rb new file mode 100644 index 0000000..d5670e0 --- /dev/null +++ b/config/metrics.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2025, by Samuel Williams. + +def prepare + require "metrics/provider/async/container" +end diff --git a/config/sus.rb b/config/sus.rb index ced7124..e5fc866 100644 --- a/config/sus.rb +++ b/config/sus.rb @@ -7,3 +7,14 @@ include Covered::Sus ENV["CONSOLE_LEVEL"] ||= "fatal" +ENV["METRICS_BACKEND"] ||= "metrics/backend/test" + +def prepare_instrumentation! + require "metrics" +end + +def before_tests(...) + prepare_instrumentation! + + super +end diff --git a/lib/async/container/generic.rb b/lib/async/container/generic.rb index b6bcaf2..4e67774 100644 --- a/lib/async/container/generic.rb +++ b/lib/async/container/generic.rb @@ -145,6 +145,13 @@ def stop(timeout = true) @running = true end + protected def health_check_failed!(child, age_clock, health_check_timeout) + Console.warn(self, "Child failed health check!", child: child, age: age_clock.total, health_check_timeout: health_check_timeout) + + # If the child has failed the health check, we assume the worst and kill it immediately: + child.kill! + end + # Spawn a child instance into the container. # @parameter name [String] The name of the child instance. # @parameter restart [Boolean] Whether to restart the child instance if it fails. @@ -176,9 +183,7 @@ def spawn(name: nil, restart: false, key: nil, health_check_timeout: nil, &block case message when :health_check! if health_check_timeout&.<(age_clock.total) - Console.warn(self, "Child failed health check!", child: child, age: age_clock.total, health_check_timeout: health_check_timeout) - # If the child has failed the health check, we assume the worst and kill it immediately: - child.kill! + health_check_failed!(child, age_clock, health_check_timeout) end else state.update(message) diff --git a/lib/metrics/provider/async/container.rb b/lib/metrics/provider/async/container.rb new file mode 100644 index 0000000..80d07c2 --- /dev/null +++ b/lib/metrics/provider/async/container.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2025, by Samuel Williams. + +require_relative "container/generic" diff --git a/lib/metrics/provider/async/container/generic.rb b/lib/metrics/provider/async/container/generic.rb new file mode 100644 index 0000000..aba27dd --- /dev/null +++ b/lib/metrics/provider/async/container/generic.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# Released under the MIT License. +# Copyright, 2025, by Samuel Williams. + +require_relative "../../../../async/container/generic" +require "metrics/provider" + +Metrics::Provider(Async::Container::Generic) do + ASYNC_CONTAINER_GENERIC_HEALTH_CHECK_FAILED = Metrics.metric("async.container.generic.health_check_failed", :counter, description: "The number of health checks that failed.") + + protected def health_check_failed!(child, age_clock, health_check_timeout) + ASYNC_CONTAINER_GENERIC_HEALTH_CHECK_FAILED.emit(1) + + super + end +end From 96e63082082c50b451a00e970ccb41610503295e Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Fri, 7 Mar 2025 20:16:36 +1300 Subject: [PATCH 2/2] Minor improvements/fixes. --- examples/health_check/test.rb | 3 ++- gems.rb | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/health_check/test.rb b/examples/health_check/test.rb index 9e92baa..70c6bbe 100755 --- a/examples/health_check/test.rb +++ b/examples/health_check/test.rb @@ -5,7 +5,8 @@ # Copyright, 2022, by Anton Sozontov. # Copyright, 2024, by Samuel Williams. -require "../../lib/async/container/controller" +require "metrics" +require_relative "../../lib/async/container/controller" NAMES = [ "Cupcake", "Donut", "Eclair", "Froyo", "Gingerbread", "Honeycomb", "Ice Cream Sandwich", "Jelly Bean", "KitKat", "Lollipop", "Marshmallow", "Nougat", "Oreo", "Pie", "Apple Tart" diff --git a/gems.rb b/gems.rb index 725e802..086b282 100644 --- a/gems.rb +++ b/gems.rb @@ -21,6 +21,8 @@ gem "decode" gem "rubocop" + gem "metrics" + gem "bake-test" gem "bake-test-external" end