From e74c999b545c41641f7a92f836d0e8f49e248dc7 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Tue, 11 Jun 2019 18:25:07 -0400 Subject: [PATCH 01/49] WIP: Move CircuitBreaker implementation to shared memory --- ext/semian/circuit_breaker.c | 67 ++++++++++++++++++++++++ ext/semian/circuit_breaker.h | 12 +++++ ext/semian/resource.c | 22 +------- ext/semian/semian.c | 3 ++ ext/semian/semian.h | 2 + ext/semian/simple_integer.c | 98 +++++++++++++++++++++++++++++++++++ ext/semian/simple_integer.h | 15 ++++++ ext/semian/sysv_semaphores.c | 35 +++---------- ext/semian/sysv_semaphores.h | 12 ++--- ext/semian/types.h | 19 +++++++ ext/semian/util.c | 41 +++++++++++++++ ext/semian/util.h | 11 ++++ lib/semian/circuit_breaker.rb | 6 ++- lib/semian/simple_integer.rb | 6 ++- test/resource_test.rb | 7 +++ test/simple_integer_test.rb | 4 +- 16 files changed, 299 insertions(+), 61 deletions(-) create mode 100644 ext/semian/circuit_breaker.c create mode 100644 ext/semian/circuit_breaker.h create mode 100644 ext/semian/simple_integer.c create mode 100644 ext/semian/simple_integer.h create mode 100644 ext/semian/util.c create mode 100644 ext/semian/util.h diff --git a/ext/semian/circuit_breaker.c b/ext/semian/circuit_breaker.c new file mode 100644 index 00000000..0f7f6f83 --- /dev/null +++ b/ext/semian/circuit_breaker.c @@ -0,0 +1,67 @@ +#include "circuit_breaker.h" + +#include +#include +#include +#include "types.h" +#include "util.h" + +static const rb_data_type_t semian_circuit_breaker_type; + +void Init_CircuitBreaker() { + printf("[DEBUG] Init_CircuitBreaker\n"); + + VALUE cSemian = rb_const_get(rb_cObject, rb_intern("Semian")); + VALUE cCircuitBreaker = rb_const_get(cSemian, rb_intern("CircuitBreaker")); + + rb_define_alloc_func(cCircuitBreaker, semian_circuit_breaker_alloc); + rb_define_method(cCircuitBreaker, "initialize_circuit_breaker", semian_circuit_breaker_initialize, 1); +} + +VALUE semian_circuit_breaker_alloc(VALUE klass) +{ + printf("[DEBUG] semian_circuit_breaker_alloc\n"); + + semian_circuit_breaker_t *res; + VALUE obj = TypedData_Make_Struct(klass, semian_circuit_breaker_t, &semian_circuit_breaker_type, res); + return obj; +} + +VALUE semian_circuit_breaker_initialize(VALUE self, VALUE id) +{ + const char *c_id_str = check_id_arg(id); + + printf("[DEBUG] semian_circuit_breaker_initialize('%s')\n", c_id_str); + + // Build semian resource structure + semian_circuit_breaker_t *res = NULL; + TypedData_Get_Struct(self, semian_circuit_breaker_t, &semian_circuit_breaker_type, res); + + // Initialize the semaphore set + // initialize_semaphore_set(res, c_id_str, c_permissions, c_tickets, c_quota); + res->name = strdup(c_id_str); + + key_t key = generate_key(res->name); + printf("[DEBUG] Creating shared memory for '%s' (id %u)\n", res->name, key); + const int permissions = 0664; + int shmid = shmget(key, 1024, IPC_CREAT | permissions); + if (shmid == -1) { + rb_raise(rb_eArgError, "could not create shared memory (%s)", strerror(errno)); + } + + printf("[DEBUG] Getting shared memory (id %u)\n", shmid); + void *val = shmat(shmid, NULL, 0); + if (val == (void*)-1) { + rb_raise(rb_eArgError, "could not get shared memory (%s)", strerror(errno)); + } + + semian_circuit_breaker_shared_t *data = (semian_circuit_breaker_shared_t*)val; + if (data == NULL) { + rb_raise(rb_eArgError, "could not get shared memory (%s)", strerror(errno)); + } + + printf("[DEBUG] successes = %d\n", data->successes); + data->successes = 0; + + return self; +} diff --git a/ext/semian/circuit_breaker.h b/ext/semian/circuit_breaker.h new file mode 100644 index 00000000..0bbb5923 --- /dev/null +++ b/ext/semian/circuit_breaker.h @@ -0,0 +1,12 @@ +#ifndef CIRCUIT_BREAKER_H +#define CIRCUIT_BREAKER_H + +#include + +void Init_CircuitBreaker(); + +VALUE semian_circuit_breaker_alloc(VALUE klass); +VALUE semian_circuit_breaker_initialize(VALUE self, VALUE id); +VALUE semian_circuit_breaker_successes(VALUE self); + +#endif // CIRCUIT_BREAKER_H \ No newline at end of file diff --git a/ext/semian/resource.c b/ext/semian/resource.c index 0eaeec5d..feed7a2d 100644 --- a/ext/semian/resource.c +++ b/ext/semian/resource.c @@ -1,5 +1,7 @@ #include "resource.h" +#include "util.h" + static VALUE cleanup_semian_resource_acquire(VALUE self); @@ -15,9 +17,6 @@ check_tickets_arg(VALUE tickets); static long check_permissions_arg(VALUE permissions); -static const -char *check_id_arg(VALUE id); - static double check_default_timeout_arg(VALUE default_timeout); @@ -314,23 +313,6 @@ check_tickets_arg(VALUE tickets) return c_tickets; } -static const char* -check_id_arg(VALUE id) -{ - const char *c_id_str = NULL; - - if (TYPE(id) != T_SYMBOL && TYPE(id) != T_STRING) { - rb_raise(rb_eTypeError, "id must be a symbol or string"); - } - if (TYPE(id) == T_SYMBOL) { - c_id_str = rb_id2name(rb_to_id(id)); - } else if (TYPE(id) == T_STRING) { - c_id_str = RSTRING_PTR(id); - } - - return c_id_str; -} - static double check_default_timeout_arg(VALUE default_timeout) { diff --git a/ext/semian/semian.c b/ext/semian/semian.c index f4d0c3ea..0c6d0337 100644 --- a/ext/semian/semian.c +++ b/ext/semian/semian.c @@ -64,4 +64,7 @@ void Init_semian() /* Maximum number of tickets available on this system. */ rb_define_const(cSemian, "MAX_TICKETS", INT2FIX(system_max_semaphore_count)); + + Init_SimpleInteger(); + Init_CircuitBreaker(); } diff --git a/ext/semian/semian.h b/ext/semian/semian.h index 55090931..65fdd588 100644 --- a/ext/semian/semian.h +++ b/ext/semian/semian.h @@ -7,7 +7,9 @@ Implements Init_semian, which is used as C/Ruby entrypoint. #ifndef SEMIAN_H #define SEMIAN_H +#include "circuit_breaker.h" #include "resource.h" +#include "simple_integer.h" void Init_semian(); diff --git a/ext/semian/simple_integer.c b/ext/semian/simple_integer.c new file mode 100644 index 00000000..bdba4629 --- /dev/null +++ b/ext/semian/simple_integer.c @@ -0,0 +1,98 @@ +#include "simple_integer.h" + +#include +#include +#include +#include "types.h" +#include "util.h" + +static const rb_data_type_t semian_simple_integer_type; + +static const char* get_string(VALUE obj) { + if (RB_TYPE_P(obj, T_STRING)) { + return RSTRING_PTR(obj); + } else if (RB_TYPE_P(obj, T_SYMBOL)) { + return rb_id2name(SYM2ID(obj)); + } + + rb_raise(rb_eArgError, "could not convert object to string"); + return NULL; +} + +static int* get_value(VALUE self) { + const char* name = get_string(rb_iv_get(self, "@name")); + if (name == NULL) { + rb_raise(rb_eArgError, "could not get object name"); + } + + key_t key = generate_key(name); + + const int permissions = 0664; + int shmid = shmget(key, sizeof(int), IPC_CREAT | permissions); + if (shmid == -1) { + rb_raise(rb_eArgError, "could not create shared memory (%s)", strerror(errno)); + } + + void *val = shmat(shmid, NULL, 0); + if (val == (void*)-1) { + rb_raise(rb_eArgError, "could not get shared memory (%s)", strerror(errno)); + } + + return (int*)val; +} + +void Init_SimpleInteger() +{ + VALUE cSemian = rb_const_get(rb_cObject, rb_intern("Semian")); + VALUE cSimple = rb_const_get(cSemian, rb_intern("Simple")); + VALUE cSimpleInteger = rb_const_get(cSimple, rb_intern("Integer")); + + rb_define_alloc_func(cSimpleInteger, semian_simple_integer_alloc); + rb_define_method(cSimpleInteger, "initialize_simple_integer", semian_simple_integer_initialize, 0); + rb_define_method(cSimpleInteger, "increment", semian_simple_integer_increment, 1); + rb_define_method(cSimpleInteger, "reset", semian_simple_integer_reset, 0); + rb_define_method(cSimpleInteger, "value", semian_simple_integer_value_get, 0); + rb_define_method(cSimpleInteger, "value=", semian_simple_integer_value_set, 1); +} + +VALUE semian_simple_integer_alloc(VALUE klass) +{ + semian_simple_integer_t *res; + VALUE obj = TypedData_Make_Struct(klass, semian_simple_integer_t, &semian_simple_integer_type, res); + return obj; +} + + +VALUE semian_simple_integer_initialize(VALUE self) +{ + int* data = get_value(self); + *data = 0; + + return self; +} + +VALUE semian_simple_integer_increment(VALUE self, VALUE val) { + int *data = get_value(self); + *data += rb_num2int(val); + + return RB_INT2NUM(*data); +} + +VALUE semian_simple_integer_reset(VALUE self) { + int *data = get_value(self); + *data = 0; + + return RB_INT2NUM(*data); +} + +VALUE semian_simple_integer_value_get(VALUE self) { + int *data = get_value(self); + return RB_INT2NUM(*data); +} + +VALUE semian_simple_integer_value_set(VALUE self, VALUE val) { + int *data = get_value(self); + *data = RB_NUM2INT(val); + + return RB_INT2NUM(*data); +} diff --git a/ext/semian/simple_integer.h b/ext/semian/simple_integer.h new file mode 100644 index 00000000..5a7f5ff0 --- /dev/null +++ b/ext/semian/simple_integer.h @@ -0,0 +1,15 @@ +#ifndef EXT_SEMIAN_SIMPLE_INTEGER_H +#define EXT_SEMIAN_SIMPLE_INTEGER_H + +#include + +void Init_SimpleInteger(); + +VALUE semian_simple_integer_alloc(VALUE klass); +VALUE semian_simple_integer_initialize(VALUE self); +VALUE semian_simple_integer_increment(VALUE self, VALUE val); +VALUE semian_simple_integer_reset(VALUE self); +VALUE semian_simple_integer_value_get(VALUE self); +VALUE semian_simple_integer_value_set(VALUE self, VALUE val); + +#endif // EXT_SEMIAN_SIMPLE_INTEGER_H \ No newline at end of file diff --git a/ext/semian/sysv_semaphores.c b/ext/semian/sysv_semaphores.c index bed7e4f3..755a4d47 100644 --- a/ext/semian/sysv_semaphores.c +++ b/ext/semian/sysv_semaphores.c @@ -1,8 +1,7 @@ #include "sysv_semaphores.h" -#include -static key_t -generate_key(const char *name); +#include +#include "util.h" static void * acquire_semaphore(void *p); @@ -30,7 +29,6 @@ raise_semian_syscall_error(const char *syscall, int error_num) void initialize_semaphore_set(semian_resource_t* res, const char* id_str, long permissions, int tickets, double quota) { - res->key = generate_key(id_str); res->strkey = (char*) malloc((2 /*for 0x*/+ sizeof(uint64_t) /*actual key*/+ 1 /*null*/) * sizeof(char)); sprintf(res->strkey, "0x%08x", (unsigned int) res->key); @@ -53,6 +51,10 @@ initialize_semaphore_set(semian_resource_t* res, const char* id_str, long permis } } +# if DEBUG + printf("[DEBUG] Init semaphore '%s' (key %s) to sem_id %d\n", res->name, res->strkey, res->sem_id); +# endif + set_semaphore_permissions(res->sem_id, permissions); /* @@ -181,31 +183,6 @@ acquire_semaphore(void *p) return NULL; } -static key_t -generate_key(const char *name) -{ - char semset_size_key[20]; - char *uniq_id_str; - - // It is necessary for the cardinatily of the semaphore set to be part of the key - // or else sem_get will complain that we have requested an incorrect number of sems - // for the desired key, and have changed the number of semaphores for a given key - sprintf(semset_size_key, "_NUM_SEMS_%d", SI_NUM_SEMAPHORES); - uniq_id_str = malloc(strlen(name)+strlen(semset_size_key)+1); - strcpy(uniq_id_str, name); - strcat(uniq_id_str, semset_size_key); - - union { - unsigned char str[SHA_DIGEST_LENGTH]; - key_t key; - } digest; - SHA1((const unsigned char *) uniq_id_str, strlen(uniq_id_str), digest.str); - free(uniq_id_str); - /* TODO: compile-time assertion that sizeof(key_t) > SHA_DIGEST_LENGTH */ - return digest.key; -} - - static void initialize_new_semaphore_values(int sem_id, long permissions) { diff --git a/ext/semian/sysv_semaphores.h b/ext/semian/sysv_semaphores.h index c3f2d106..205a6cbb 100644 --- a/ext/semian/sysv_semaphores.h +++ b/ext/semian/sysv_semaphores.h @@ -11,7 +11,6 @@ and functions associated directly weth semops. #include #include -#include #include #include #include @@ -110,11 +109,12 @@ acquire_semaphore_without_gvl(void *p); static inline void print_sem_vals(int sem_id) { - printf("lock %d, tickets: %d configured: %d, registered workers %d\n", - get_sem_val(sem_id, SI_SEM_LOCK), - get_sem_val(sem_id, SI_SEM_TICKETS), - get_sem_val(sem_id, SI_SEM_CONFIGURED_TICKETS), - get_sem_val(sem_id, SI_SEM_REGISTERED_WORKERS) + printf("[DEBUG] sem_id: %d, lock: %d, tickets: %d configured: %d, registered workers %d\n", + sem_id, + get_sem_val(sem_id, SI_SEM_LOCK), + get_sem_val(sem_id, SI_SEM_TICKETS), + get_sem_val(sem_id, SI_SEM_CONFIGURED_TICKETS), + get_sem_val(sem_id, SI_SEM_REGISTERED_WORKERS) ); } #endif diff --git a/ext/semian/types.h b/ext/semian/types.h index de0254ec..338bdf02 100644 --- a/ext/semian/types.h +++ b/ext/semian/types.h @@ -38,4 +38,23 @@ typedef struct { long wait_time; } semian_resource_t; +// Internal circuit breaker structure +typedef struct { + int sem_id; + uint64_t key; + char *strkey; + char *name; +} semian_circuit_breaker_t; + +// Shared circuit breaker structure +typedef struct { + int successes; +} semian_circuit_breaker_shared_t; + +// Internal simple integer structure +typedef struct { + uint64_t key; + int val; +} semian_simple_integer_t; + #endif // SEMIAN_TYPES_H diff --git a/ext/semian/util.c b/ext/semian/util.c new file mode 100644 index 00000000..f7f391c8 --- /dev/null +++ b/ext/semian/util.c @@ -0,0 +1,41 @@ +#include "util.h" + +const char* check_id_arg(VALUE id) +{ + if (TYPE(id) != T_SYMBOL && TYPE(id) != T_STRING) { + rb_raise(rb_eTypeError, "id must be a symbol or string"); + } + + const char *c_id_str = NULL; + if (TYPE(id) == T_SYMBOL) { + c_id_str = rb_id2name(rb_to_id(id)); + } else if (TYPE(id) == T_STRING) { + c_id_str = RSTRING_PTR(id); + } + + return c_id_str; +} + +key_t generate_key(const char *name) +{ + char semset_size_key[128]; + char *uniq_id_str; + + // It is necessary for the cardinatily of the semaphore set to be part of the key + // or else sem_get will complain that we have requested an incorrect number of sems + // for the desired key, and have changed the number of semaphores for a given key + const int NUM_SEMAPHORES = 4; + sprintf(semset_size_key, "_NUM_SEMS_%d", NUM_SEMAPHORES); + uniq_id_str = malloc(strlen(name) + strlen(semset_size_key) + 1); + strcpy(uniq_id_str, name); + strcat(uniq_id_str, semset_size_key); + + union { + unsigned char str[SHA_DIGEST_LENGTH]; + key_t key; + } digest; + SHA1((const unsigned char *) uniq_id_str, strlen(uniq_id_str), digest.str); + free(uniq_id_str); + /* TODO: compile-time assertion that sizeof(key_t) > SHA_DIGEST_LENGTH */ + return digest.key; +} \ No newline at end of file diff --git a/ext/semian/util.h b/ext/semian/util.h new file mode 100644 index 00000000..e2bd711f --- /dev/null +++ b/ext/semian/util.h @@ -0,0 +1,11 @@ +#ifndef EXT_SEMIAN_UTIL_H +#define EXT_SEMIAN_UTIL_H + +#include +#include + +const char* check_id_arg(VALUE id); + +key_t generate_key(const char *name); + +#endif // EXT_SEMIAN_UTIL_H diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index a19a0f36..bfe7bdd5 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -9,6 +9,8 @@ class CircuitBreaker #:nodoc: def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, implementation:, half_open_resource_timeout: nil) @name = name.to_sym + initialize_circuit_breaker(@name) + @success_count_threshold = success_threshold @error_count_threshold = error_threshold @error_timeout = error_timeout @@ -16,7 +18,7 @@ def initialize(name, exceptions:, success_threshold:, error_threshold:, @half_open_resource_timeout = half_open_resource_timeout @errors = implementation::SlidingWindow.new(max_size: @error_count_threshold) - @successes = implementation::Integer.new + @successes = implementation::Integer.new(name) @state = implementation::State.new reset @@ -62,7 +64,7 @@ def mark_failed(error) def mark_success return unless half_open? - @successes.increment + @successes.increment(1) transition_to_close if success_threshold_reached? end diff --git a/lib/semian/simple_integer.rb b/lib/semian/simple_integer.rb index 51bc498b..3b912fbf 100644 --- a/lib/semian/simple_integer.rb +++ b/lib/semian/simple_integer.rb @@ -5,11 +5,13 @@ module Simple class Integer #:nodoc: attr_accessor :value - def initialize + def initialize(name) + @name = name + initialize_simple_integer if respond_to?(:initialize_simple_integer) reset end - def increment(val = 1) + def increment(val) @value += val end diff --git a/test/resource_test.rb b/test/resource_test.rb index ad837852..09034980 100644 --- a/test/resource_test.rb +++ b/test/resource_test.rb @@ -1,5 +1,7 @@ require 'test_helper' +require 'objspace' + class TestResource < Minitest::Test include ResourceHelper @@ -490,6 +492,11 @@ def test_multiple_register_with_fork assert_equal 0, timeouts end + def test_memsize + r = create_resource :testing, tickets: 1 + puts "mkipper: Resource size is #{ObjectSpace.memsize_of(r)} byte(s)" + end + def create_resource(*args) @resources ||= [] resource = Semian::Resource.new(*args) diff --git a/test/simple_integer_test.rb b/test/simple_integer_test.rb index 2df82439..aa03eeb1 100644 --- a/test/simple_integer_test.rb +++ b/test/simple_integer_test.rb @@ -2,7 +2,7 @@ class TestSimpleInteger < Minitest::Test def setup - @integer = ::Semian::ThreadSafe::Integer.new + @integer = ::Semian::ThreadSafe::Integer.new(:simple_integer) end def teardown @@ -26,7 +26,7 @@ def test_access_value def test_increment @integer.increment(4) assert_equal(4, @integer.value) - @integer.increment + @integer.increment(1) assert_equal(5, @integer.value) @integer.increment(-2) assert_equal(3, @integer.value) From e89c2c50e8e4582ea31d7e6bcf3bde4af8e29bd1 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Wed, 12 Jun 2019 10:14:08 -0400 Subject: [PATCH 02/49] WIP: Move Simple::State to shared memory --- lib/semian/circuit_breaker.rb | 5 +- lib/semian/circuit_breaker.rb.orig | 171 +++++++++++++++++++++++++++++ lib/semian/simple_state.rb | 27 +++-- test/simple_state_test.rb | 11 +- 4 files changed, 198 insertions(+), 16 deletions(-) create mode 100644 lib/semian/circuit_breaker.rb.orig diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index bfe7bdd5..2b68602d 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -18,8 +18,9 @@ def initialize(name, exceptions:, success_threshold:, error_threshold:, @half_open_resource_timeout = half_open_resource_timeout @errors = implementation::SlidingWindow.new(max_size: @error_count_threshold) - @successes = implementation::Integer.new(name) - @state = implementation::State.new + @successes = implementation::Integer.new("#{name}_successes") + state_val = implementation::Integer.new("#{name}_state") + @state = implementation::State.new(state_val) reset end diff --git a/lib/semian/circuit_breaker.rb.orig b/lib/semian/circuit_breaker.rb.orig new file mode 100644 index 00000000..5f812784 --- /dev/null +++ b/lib/semian/circuit_breaker.rb.orig @@ -0,0 +1,171 @@ +module Semian + class CircuitBreaker #:nodoc: + extend Forwardable + + def_delegators :@state, :closed?, :open?, :half_open? + + attr_reader :name, :half_open_resource_timeout, :error_timeout, :state, :last_error + + def initialize(name, exceptions:, success_threshold:, error_threshold:, + error_timeout:, implementation:, half_open_resource_timeout: nil) + @name = name.to_sym + initialize_circuit_breaker(@name) + + @success_count_threshold = success_threshold + @error_count_threshold = error_threshold + @error_timeout = error_timeout + @exceptions = exceptions + @half_open_resource_timeout = half_open_resource_timeout + + @errors = implementation::SlidingWindow.new(max_size: @error_count_threshold) +<<<<<<< HEAD + @successes = implementation::Integer.new(name) + @state = implementation::State.new + + reset +======= + @successes = implementation::Integer.new("#{name}_successes") + state_val = implementation::Integer.new("#{name}_state") + @state = implementation::State.new(state_val) +>>>>>>> WIP: Move Simple::State to shared memory + end + + def acquire(resource = nil, &block) + return yield if disabled? + transition_to_half_open if transition_to_half_open? + + raise OpenCircuitError unless request_allowed? + + result = nil + begin + result = maybe_with_half_open_resource_timeout(resource, &block) + rescue *@exceptions => error + if !error.respond_to?(:marks_semian_circuits?) || error.marks_semian_circuits? + mark_failed(error) + end + raise error + else + mark_success + end + result + end + + def transition_to_half_open? + open? && error_timeout_expired? && !half_open? + end + + def request_allowed? + closed? || half_open? || transition_to_half_open? + end + + def mark_failed(error) + push_error(error) + push_time(@errors) + if closed? + transition_to_open if error_threshold_reached? + elsif half_open? + transition_to_open + end + end + + def mark_success + return unless half_open? + @successes.increment(1) + transition_to_close if success_threshold_reached? + end + + def reset + @errors.clear + @successes.reset + transition_to_close + end + + def destroy + @errors.destroy + @successes.destroy + @state.destroy + end + + def in_use? + return false if error_timeout_expired? + @errors.size > 0 + end + + private + + def transition_to_close + notify_state_transition(:closed) + log_state_transition(:closed) + @state.close! + @errors.clear + end + + def transition_to_open + notify_state_transition(:open) + log_state_transition(:open) + @state.open! + end + + def transition_to_half_open + notify_state_transition(:half_open) + log_state_transition(:half_open) + @state.half_open! + @successes.reset + end + + def success_threshold_reached? + @successes.value >= @success_count_threshold + end + + def error_threshold_reached? + @errors.size == @error_count_threshold + end + + def error_timeout_expired? + last_error_time = @errors.last + return false unless last_error_time + Time.at(last_error_time) + @error_timeout < Time.now + end + + def push_error(error) + @last_error = error + end + + def push_time(window, time: Time.now) + window.reject! { |err_time| err_time + @error_timeout < time.to_i } + window << time.to_i + end + + def log_state_transition(new_state) + return if @state.nil? || new_state == @state.value + + str = "[#{self.class.name}] State transition from #{@state.value} to #{new_state}." + str << " success_count=#{@successes.value} error_count=#{@errors.size}" + str << " success_count_threshold=#{@success_count_threshold} error_count_threshold=#{@error_count_threshold}" + str << " error_timeout=#{@error_timeout} error_last_at=\"#{@errors.last}\"" + str << " name=\"#{@name}\"" + Semian.logger.info(str) + end + + def notify_state_transition(new_state) + Semian.notify(:state_change, self, nil, nil, state: new_state) + end + + def disabled? + ENV['SEMIAN_CIRCUIT_BREAKER_DISABLED'] || ENV['SEMIAN_DISABLED'] + end + + def maybe_with_half_open_resource_timeout(resource, &block) + result = + if half_open? && @half_open_resource_timeout && resource.respond_to?(:with_resource_timeout) + resource.with_resource_timeout(@half_open_resource_timeout) do + block.call + end + else + block.call + end + + result + end + end +end diff --git a/lib/semian/simple_state.rb b/lib/semian/simple_state.rb index 322d333d..40d8041e 100644 --- a/lib/semian/simple_state.rb +++ b/lib/semian/simple_state.rb @@ -1,34 +1,43 @@ module Semian module Simple class State #:nodoc: - def initialize + extend Forwardable + + def_delegators :@value, :value + + # State constants. Looks like a flag, but is actually an enum. + UNKNOWN = 0x0 + OPEN = 0x1 + CLOSED = 0x2 + HALF_OPEN = 0x4 + + def initialize(value) + @value = value reset end - attr_reader :value - def open? - value == :open + @value.value == OPEN end def closed? - value == :closed + @value.value == CLOSED end def half_open? - value == :half_open + @value.value == HALF_OPEN end def open! - @value = :open + @value.value = OPEN end def close! - @value = :closed + @value.value = CLOSED end def half_open! - @value = :half_open + @value.value = HALF_OPEN end def reset diff --git a/test/simple_state_test.rb b/test/simple_state_test.rb index 0da9dd53..8dc3b4aa 100644 --- a/test/simple_state_test.rb +++ b/test/simple_state_test.rb @@ -2,7 +2,8 @@ class TestSimpleEnum < Minitest::Test def setup - @state = ::Semian::ThreadSafe::State.new + state_val = ::Semian::ThreadSafe::Integer.new(:test_simple_enum) + @state = ::Semian::ThreadSafe::State.new(state_val) end def teardown @@ -17,25 +18,25 @@ def test_start_closed? def test_open @state.open! assert @state.open? - assert_equal @state.value, :open + assert_equal ::Semian::Simple::State::OPEN, @state.value end def test_close @state.close! assert @state.closed? - assert_equal @state.value, :closed + assert_equal ::Semian::Simple::State::CLOSED, @state.value end def test_half_open @state.half_open! assert @state.half_open? - assert_equal @state.value, :half_open + assert_equal ::Semian::Simple::State::HALF_OPEN, @state.value end def test_reset @state.reset assert @state.closed? - assert_equal @state.value, :closed + assert_equal ::Semian::Simple::State::CLOSED, @state.value end end From bb0e96e8dceab9ed756ff30ebdfd57bef6c6eac9 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Wed, 12 Jun 2019 10:40:54 -0400 Subject: [PATCH 03/49] WIP: Working Simple::Integer and Simple::State --- ext/semian/semian.c | 1 + ext/semian/semian.h | 1 + ext/semian/simple_integer.c | 5 ++++- ext/semian/sliding_window.c | 4 ++++ ext/semian/sliding_window.h | 8 ++++++++ lib/semian/circuit_breaker.rb | 2 +- lib/semian/simple_state.rb | 19 +++++++++++++++++-- 7 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 ext/semian/sliding_window.c create mode 100644 ext/semian/sliding_window.h diff --git a/ext/semian/semian.c b/ext/semian/semian.c index 0c6d0337..1b730473 100644 --- a/ext/semian/semian.c +++ b/ext/semian/semian.c @@ -66,5 +66,6 @@ void Init_semian() rb_define_const(cSemian, "MAX_TICKETS", INT2FIX(system_max_semaphore_count)); Init_SimpleInteger(); + Init_SlidingWindow(); Init_CircuitBreaker(); } diff --git a/ext/semian/semian.h b/ext/semian/semian.h index 65fdd588..c38636e0 100644 --- a/ext/semian/semian.h +++ b/ext/semian/semian.h @@ -10,6 +10,7 @@ Implements Init_semian, which is used as C/Ruby entrypoint. #include "circuit_breaker.h" #include "resource.h" #include "simple_integer.h" +#include "sliding_window.h" void Init_semian(); diff --git a/ext/semian/simple_integer.c b/ext/semian/simple_integer.c index bdba4629..0aee4443 100644 --- a/ext/semian/simple_integer.c +++ b/ext/semian/simple_integer.c @@ -92,7 +92,10 @@ VALUE semian_simple_integer_value_get(VALUE self) { VALUE semian_simple_integer_value_set(VALUE self, VALUE val) { int *data = get_value(self); - *data = RB_NUM2INT(val); + + // TODO(michaelkipper): Check for respond_to?(:to_i) before calling. + VALUE to_i = rb_funcall(val, rb_intern("to_i"), 0); + *data = RB_NUM2INT(to_i); return RB_INT2NUM(*data); } diff --git a/ext/semian/sliding_window.c b/ext/semian/sliding_window.c new file mode 100644 index 00000000..346734f8 --- /dev/null +++ b/ext/semian/sliding_window.c @@ -0,0 +1,4 @@ +#include "sliding_window.h" + +void Init_SlidingWindow() { +} diff --git a/ext/semian/sliding_window.h b/ext/semian/sliding_window.h new file mode 100644 index 00000000..bfab4789 --- /dev/null +++ b/ext/semian/sliding_window.h @@ -0,0 +1,8 @@ +#ifndef EXT_SEMIAN_SLIDING_WINDOW_H +#define EXT_SEMIAN_SLIDING_WINDOW_H + +#include "types.h" + +void Init_SlidingWindow(); + +#endif // EXT_SEMIAN_SLIDING_WINDOW_H \ No newline at end of file diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index 2b68602d..f7c9e408 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -134,7 +134,7 @@ def push_time(window, time: Time.now) def log_state_transition(new_state) return if @state.nil? || new_state == @state.value - str = "[#{self.class.name}] State transition from #{@state.value} to #{new_state}." + str = "[#{self.class.name}] State transition from #{@state} to #{new_state}." str << " success_count=#{@successes.value} error_count=#{@errors.size}" str << " success_count_threshold=#{@success_count_threshold} error_count_threshold=#{@error_count_threshold}" str << " error_timeout=#{@error_timeout} error_last_at=\"#{@errors.last}\"" diff --git a/lib/semian/simple_state.rb b/lib/semian/simple_state.rb index 40d8041e..228c2e41 100644 --- a/lib/semian/simple_state.rb +++ b/lib/semian/simple_state.rb @@ -7,8 +7,8 @@ class State #:nodoc: # State constants. Looks like a flag, but is actually an enum. UNKNOWN = 0x0 - OPEN = 0x1 - CLOSED = 0x2 + CLOSED = 0x1 + OPEN = 0x2 HALF_OPEN = 0x4 def initialize(value) @@ -47,6 +47,21 @@ def reset def destroy reset end + + def to_s + case @value.value + when UNKNOWN + "unknown" + when CLOSED + "closed" + when OPEN + "open" + when HALF_OPEN + "half_open" + else + "" + end + end end end From 1d04c0c687b80056109536f9d4e867d706b20e85 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Wed, 12 Jun 2019 15:28:13 -0400 Subject: [PATCH 04/49] WIP: Working Simple::SlidingWindow in shared memory --- ext/semian/simple_integer.c | 13 +- ext/semian/sliding_window.c | 208 ++++++++++++++++++++++++++++ ext/semian/sliding_window.h | 13 +- ext/semian/types.h | 17 ++- ext/semian/util.c | 13 +- ext/semian/util.h | 2 + lib/semian/circuit_breaker.rb | 6 +- lib/semian/simple_sliding_window.rb | 24 +++- test/adapter_test.rb | 2 +- test/circuit_breaker_test.rb | 8 ++ test/redis_test.rb | 2 + test/resource_test.rb | 4 +- test/simple_sliding_window_test.rb | 24 +++- 13 files changed, 307 insertions(+), 29 deletions(-) diff --git a/ext/semian/simple_integer.c b/ext/semian/simple_integer.c index 0aee4443..2966d247 100644 --- a/ext/semian/simple_integer.c +++ b/ext/semian/simple_integer.c @@ -8,19 +8,8 @@ static const rb_data_type_t semian_simple_integer_type; -static const char* get_string(VALUE obj) { - if (RB_TYPE_P(obj, T_STRING)) { - return RSTRING_PTR(obj); - } else if (RB_TYPE_P(obj, T_SYMBOL)) { - return rb_id2name(SYM2ID(obj)); - } - - rb_raise(rb_eArgError, "could not convert object to string"); - return NULL; -} - static int* get_value(VALUE self) { - const char* name = get_string(rb_iv_get(self, "@name")); + const char* name = to_s(rb_iv_get(self, "@name")); if (name == NULL) { rb_raise(rb_eArgError, "could not get object name"); } diff --git a/ext/semian/sliding_window.c b/ext/semian/sliding_window.c index 346734f8..3615f1eb 100644 --- a/ext/semian/sliding_window.c +++ b/ext/semian/sliding_window.c @@ -1,4 +1,212 @@ #include "sliding_window.h" +#include +#include +#include +#include "util.h" + +static const rb_data_type_t semian_simple_sliding_window_type; + +static semian_simple_sliding_window_shared_t* get_window(uint64_t key) { + const int permissions = 0664; + int shmid = shmget(key, sizeof(int), IPC_CREAT | permissions); + if (shmid == -1) { + rb_raise(rb_eArgError, "could not create shared memory (%s)", strerror(errno)); + } + + void *val = shmat(shmid, NULL, 0); + if (val == (void*)-1) { + rb_raise(rb_eArgError, "could not get shared memory (%s)", strerror(errno)); + } + + return (semian_simple_sliding_window_shared_t*)val; +} + +static int check_max_size_arg(VALUE max_size) +{ + int retval = -1; + switch (TYPE(max_size)) { + case T_NIL: + retval = SLIDING_WINDOW_MAX_SIZE; break; + case T_FLOAT: + rb_warn("semian sliding window max_size is a float, converting to fixnum"); + retval = (int)(RFLOAT_VALUE(max_size)); break; + default: + retval = RB_NUM2INT(max_size); break; + } + + if (retval <= 0) { + rb_raise(rb_eArgError, "max_size must be greater than zero"); + } else if (retval > SLIDING_WINDOW_MAX_SIZE) { + rb_raise(rb_eArgError, "max_size must be less than %d", SLIDING_WINDOW_MAX_SIZE); + } + + return retval; +} + void Init_SlidingWindow() { + printf("[DEBUG] Init_SlidingWindow\n"); + + VALUE cSemian = rb_const_get(rb_cObject, rb_intern("Semian")); + VALUE cSimple = rb_const_get(cSemian, rb_intern("Simple")); + VALUE cSlidingWindow = rb_const_get(cSimple, rb_intern("SlidingWindow")); + + rb_define_alloc_func(cSlidingWindow, semian_simple_sliding_window_alloc); + rb_define_method(cSlidingWindow, "initialize_sliding_window", semian_simple_sliding_window_initialize, 2); + rb_define_method(cSlidingWindow, "size", semian_simple_sliding_window_size, 0); + rb_define_method(cSlidingWindow, "max_size", semian_simple_sliding_window_max_size, 0); + rb_define_method(cSlidingWindow, "values", semian_simple_sliding_window_values, 0); + rb_define_method(cSlidingWindow, "last", semian_simple_sliding_window_last, 0); + rb_define_method(cSlidingWindow, "<<", semian_simple_sliding_window_push, 1); + rb_define_method(cSlidingWindow, "destroy", semian_simple_sliding_window_clear, 0); + rb_define_method(cSlidingWindow, "reject!", semian_simple_sliding_window_reject, 0); +} + +VALUE semian_simple_sliding_window_alloc(VALUE klass) { + semian_simple_sliding_window_t *res; + VALUE obj = TypedData_Make_Struct(klass, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); + return obj; +} + +VALUE semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_size) { + printf("[DEBUG] semian_simple_sliding_window_initialize\n"); + + semian_simple_sliding_window_t *res; + TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); + + const char *id_str = check_id_arg(name); + res->key = generate_key(id_str); + + semian_simple_sliding_window_shared_t *window = get_window(res->key); + window->max_size = check_max_size_arg(max_size); + window->length = 0; + window->start = 0; + window->end = 0; + + printf("[DEBUG] key:%lu addr:0x%p max_size:%d length:%d start:%d end:%d\n", res->key, window, window->max_size, window->length, window->start, window->end); + + return self; +} + +VALUE semian_simple_sliding_window_size(VALUE self) { + printf("[DEBUG] semian_simple_sliding_window_size\n"); + + semian_simple_sliding_window_t *res; + TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); + + semian_simple_sliding_window_shared_t *window = get_window(res->key); + printf("[DEBUG] key:%lu addr:0x%p max_size:%d length:%d start:%d end:%d\n", res->key, window, window->max_size, window->length, window->start, window->end); + + return RB_INT2NUM(window->length); +} + +VALUE semian_simple_sliding_window_max_size(VALUE self) { + printf("[DEBUG] semian_simple_sliding_window_max_size\n"); + + semian_simple_sliding_window_t *res; + TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); + + semian_simple_sliding_window_shared_t *window = get_window(res->key); + printf("[DEBUG] key:%lu addr:0x%p max_size:%d length:%d start:%d end:%d\n", res->key, window, window->max_size, window->length, window->start, window->end); + + return RB_INT2NUM(window->max_size); +} + +VALUE semian_simple_sliding_window_values(VALUE self) { + printf("[DEBUG] semian_simple_sliding_window_values\n"); + + semian_simple_sliding_window_t *res; + TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); + semian_simple_sliding_window_shared_t *window = get_window(res->key); + + VALUE retval = rb_ary_new_capa(window->length); + for (int i = 0; i < window->length; ++i) { + int index = (window->start + i) % window->max_size; + int value = window->data[index]; + printf("[DEBUG] i:%d index: %d value:%d max_size:%d length:%d start:%d end:%d\n", i, index, value, window->max_size, window->length, window->start, window->end); + rb_ary_store(retval, i, RB_INT2NUM(value)); + } + + return retval; +} + +VALUE semian_simple_sliding_window_last(VALUE self) { + printf("[DEBUG] semian_simple_sliding_window_last\n"); + + semian_simple_sliding_window_t *res; + TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); + semian_simple_sliding_window_shared_t *window = get_window(res->key); + + int index = (window->start + window->length - 1) % window->max_size; + printf("[DEBUG] index:%d last:%d\n", index, window->data[index]); + + return RB_INT2NUM(window->data[index]); +} + +VALUE semian_simple_sliding_window_clear(VALUE self) { + printf("[DEBUG] semian_simple_sliding_window_clear\n"); + + semian_simple_sliding_window_t *res; + TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); + semian_simple_sliding_window_shared_t *window = get_window(res->key); + + window->length = 0; + window->start = 0; + window->end = 0; + + return self; +} + +VALUE semian_simple_sliding_window_reject(VALUE self) { + printf("[DEBUG] semian_simple_sliding_window_reject\n"); + + rb_need_block(); + + semian_simple_sliding_window_t *res; + TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); + semian_simple_sliding_window_shared_t *window = get_window(res->key); + + // Store these values because we're going to be modifying the buffer. + int start = window->start; + int length = window->length; + + int cleared = 0; + for (int i = 0; i < length; ++i) { + int index = (start + i) % length; + int value = window->data[index]; + printf("[DEBUG] i:%d index: %d value:%d max_size:%d length:%d start:%d end:%d\n", i, index, value, window->max_size, window->length, window->start, window->end); + VALUE y = rb_yield(RB_INT2NUM(value)); + if (RTEST(y)) { + if (cleared++ != i) { + rb_raise(rb_eArgError, "reject! must delete monotonically"); + } + window->start = (window->start + 1) % window->length; + window->length--; + printf("[DEBUG] Removed index:%d (val:%d)\n", i, value); + } + } + + return self; +} + +VALUE semian_simple_sliding_window_push(VALUE self, VALUE value) { + printf("[DEBUG] semian_simple_sliding_window_push\n"); + + semian_simple_sliding_window_t *res; + TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); + + semian_simple_sliding_window_shared_t *window = get_window(res->key); + if (window->length == window->max_size) { + window->length--; + window->start = (window->start + 1) % window->max_size; + } + + const int index = window->end; + window->length++; + window->data[index] = RB_NUM2INT(value); + window->end = (window->end + 1) % window->max_size; + + printf("[DEBUG] Pushed val:%d index:%d max_size:%d length:%d start:%d end:%d\n", RB_NUM2INT(value), index, window->max_size, window->length, window->start, window->end); + + return self; } diff --git a/ext/semian/sliding_window.h b/ext/semian/sliding_window.h index bfab4789..083df050 100644 --- a/ext/semian/sliding_window.h +++ b/ext/semian/sliding_window.h @@ -1,8 +1,19 @@ #ifndef EXT_SEMIAN_SLIDING_WINDOW_H #define EXT_SEMIAN_SLIDING_WINDOW_H +#include #include "types.h" void Init_SlidingWindow(); -#endif // EXT_SEMIAN_SLIDING_WINDOW_H \ No newline at end of file +VALUE semian_simple_sliding_window_alloc(VALUE klass); +VALUE semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_size); +VALUE semian_simple_sliding_window_size(VALUE self); +VALUE semian_simple_sliding_window_max_size(VALUE self); +VALUE semian_simple_sliding_window_push(VALUE self, VALUE value); +VALUE semian_simple_sliding_window_values(VALUE self); +VALUE semian_simple_sliding_window_last(VALUE self); +VALUE semian_simple_sliding_window_clear(VALUE self); +VALUE semian_simple_sliding_window_reject(VALUE self); + +#endif // EXT_SEMIAN_SLIDING_WINDOW_H diff --git a/ext/semian/types.h b/ext/semian/types.h index 338bdf02..70ec5f7a 100644 --- a/ext/semian/types.h +++ b/ext/semian/types.h @@ -10,6 +10,8 @@ For custom type definitions specific to semian #include #include +#define SLIDING_WINDOW_MAX_SIZE 4096 + // For sysV semop syscals // see man semop union semun { @@ -54,7 +56,20 @@ typedef struct { // Internal simple integer structure typedef struct { uint64_t key; - int val; } semian_simple_integer_t; +// Shared simple sliding window structure +typedef struct { + int max_size; + int length; + int start; + int end; + int data[SLIDING_WINDOW_MAX_SIZE]; +} semian_simple_sliding_window_shared_t; + +// Internal simple sliding window structure +typedef struct { + uint64_t key; +} semian_simple_sliding_window_t; + #endif // SEMIAN_TYPES_H diff --git a/ext/semian/util.c b/ext/semian/util.c index f7f391c8..44b2e61e 100644 --- a/ext/semian/util.c +++ b/ext/semian/util.c @@ -38,4 +38,15 @@ key_t generate_key(const char *name) free(uniq_id_str); /* TODO: compile-time assertion that sizeof(key_t) > SHA_DIGEST_LENGTH */ return digest.key; -} \ No newline at end of file +} + +const char* to_s(VALUE obj) { + if (RB_TYPE_P(obj, T_STRING)) { + return RSTRING_PTR(obj); + } else if (RB_TYPE_P(obj, T_SYMBOL)) { + return rb_id2name(SYM2ID(obj)); + } + + rb_raise(rb_eArgError, "could not convert object to string"); + return NULL; +} diff --git a/ext/semian/util.h b/ext/semian/util.h index e2bd711f..f439e4c3 100644 --- a/ext/semian/util.h +++ b/ext/semian/util.h @@ -8,4 +8,6 @@ const char* check_id_arg(VALUE id); key_t generate_key(const char *name); +const char* to_s(VALUE obj); + #endif // EXT_SEMIAN_UTIL_H diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index f7c9e408..0dccbe88 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -17,7 +17,7 @@ def initialize(name, exceptions:, success_threshold:, error_threshold:, @exceptions = exceptions @half_open_resource_timeout = half_open_resource_timeout - @errors = implementation::SlidingWindow.new(max_size: @error_count_threshold) + @errors = implementation::SlidingWindow.new("#{name}_window", max_size: @error_count_threshold) @successes = implementation::Integer.new("#{name}_successes") state_val = implementation::Integer.new("#{name}_state") @state = implementation::State.new(state_val) @@ -26,6 +26,8 @@ def initialize(name, exceptions:, success_threshold:, error_threshold:, end def acquire(resource = nil, &block) + puts "[DEBUG] #{Time.now} - Acquiring resource '#{resource}'" + return yield if disabled? transition_to_half_open if transition_to_half_open? @@ -117,6 +119,7 @@ def error_threshold_reached? end def error_timeout_expired? + puts "[DEBUG] Checking error_timeout_expired? #{@errors.last}" last_error_time = @errors.last return false unless last_error_time Time.at(last_error_time) + @error_timeout < Time.now @@ -127,6 +130,7 @@ def push_error(error) end def push_time(window, time: Time.now) + puts "[DEBUG] push_time(#{time.to_i}) - rejecting before #{time.to_i - @error_timeout}" window.reject! { |err_time| err_time + @error_timeout < time.to_i } window << time.to_i end diff --git a/lib/semian/simple_sliding_window.rb b/lib/semian/simple_sliding_window.rb index c31cda02..15ad1e43 100644 --- a/lib/semian/simple_sliding_window.rb +++ b/lib/semian/simple_sliding_window.rb @@ -3,20 +3,32 @@ module Semian module Simple class SlidingWindow #:nodoc: - extend Forwardable - - def_delegators :@window, :size, :last - attr_reader :max_size - # A sliding window is a structure that stores the most @max_size recent timestamps # like this: if @max_size = 4, current time is 10, @window =[5,7,9,10]. # Another push of (11) at 11 sec would make @window [7,9,10,11], shifting off 5. - def initialize(max_size:) + def initialize(name, max_size:) + initialize_sliding_window(name, max_size) @max_size = max_size @window = [] end + def size + @window.size + end + + def max_size + @max_size + end + + def last + @window.last + end + + def values + @window + end + def reject!(&block) @window.reject!(&block) end diff --git a/test/adapter_test.rb b/test/adapter_test.rb index 791a03b2..01c561f0 100644 --- a/test/adapter_test.rb +++ b/test/adapter_test.rb @@ -24,7 +24,7 @@ def test_unregister skip if ENV["SKIP_FLAKY_TESTS"] client = Semian::AdapterTestClient.new(quota: 0.5) assert_nil(Semian.resources[:testing]) - resource = Semian.register(:testing, tickets: 2, error_threshold: 0, error_timeout: 0, success_threshold: 0) + resource = Semian.register(:testing, tickets: 2, error_threshold: 1, error_timeout: 0, success_threshold: 0) assert_equal(Semian.resources[:testing], resource) assert_equal 1, resource.registered_workers diff --git a/test/circuit_breaker_test.rb b/test/circuit_breaker_test.rb index be4c01c9..594d4a07 100644 --- a/test/circuit_breaker_test.rb +++ b/test/circuit_breaker_test.rb @@ -11,6 +11,8 @@ def setup rescue nil end + + puts "[DEBUG] Registering semian :testing at #{Time.now} (#{Time.now.to_i})" Semian.register(:testing, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) @resource = Semian[:testing] end @@ -40,8 +42,14 @@ def test_after_error_timeout_is_elapsed_requests_are_attempted_again end def test_until_success_threshold_is_reached_a_single_error_will_reopen_the_circuit + puts @strio.string + puts "[DEBUG] Half open circuit..." half_open_cicuit! + puts @strio.string + puts "[DEBUG] Trigger error..." trigger_error! + puts @strio.string + puts "[DEBUG] Assert circuit opened..." assert_circuit_opened assert_match(/State transition from open to half_open/, @strio.string) end diff --git a/test/redis_test.rb b/test/redis_test.rb index 1ae2dbaf..c13e5551 100644 --- a/test/redis_test.rb +++ b/test/redis_test.rb @@ -288,7 +288,9 @@ def test_circuit_breaker_on_query client.get('foo') end + puts "[DEBUG] #1 Time.now = #{Time.now}" Timecop.travel(ERROR_TIMEOUT + 1) do + puts "[DEBUG] #2 Time.now = #{Time.now}" assert_equal '2', client.get('foo') end end diff --git a/test/resource_test.rb b/test/resource_test.rb index 09034980..95487df0 100644 --- a/test/resource_test.rb +++ b/test/resource_test.rb @@ -58,7 +58,7 @@ def test_register_with_quota def test_unregister_past_0 workers = 10 - resource = Semian.register(:testing, tickets: workers * 2, error_threshold: 0, error_timeout: 0, success_threshold: 0) + resource = Semian.register(:testing, tickets: workers * 2, error_threshold: 1, error_timeout: 0, success_threshold: 0) fork_workers(count: workers, tickets: 0, timeout: 0.5, wait_for_timeout: true) do Semian.unregister(:testing) @@ -73,7 +73,7 @@ def test_unregister_past_0 def test_reset_registered_workers workers = 10 - resource = Semian.register(:testing, tickets: 1, error_threshold: 0, error_timeout: 0, success_threshold: 0) + resource = Semian.register(:testing, tickets: 1, error_threshold: 1, error_timeout: 0, success_threshold: 0) fork_workers(count: workers - 1, tickets: 0, timeout: 0.5, wait_for_timeout: true) diff --git a/test/simple_sliding_window_test.rb b/test/simple_sliding_window_test.rb index 2e10d18e..618be2e5 100644 --- a/test/simple_sliding_window_test.rb +++ b/test/simple_sliding_window_test.rb @@ -2,7 +2,7 @@ class TestSimpleSlidingWindow < Minitest::Test def setup - @sliding_window = ::Semian::ThreadSafe::SlidingWindow.new(max_size: 6) + @sliding_window = ::Semian::ThreadSafe::SlidingWindow.new(:sliding_window_test, max_size: 6) @sliding_window.clear end @@ -24,6 +24,24 @@ def test_sliding_window_edge_falloff assert_sliding_window(@sliding_window, [2, 3, 4, 5, 6, 7], 6) end + def test_sliding_window_reject + @sliding_window << 0 << 1 << 2 << 3 << 4 << 5 << 6 << 7; + assert_equal(6, @sliding_window.size) + assert_sliding_window(@sliding_window, [2, 3, 4, 5, 6, 7], 6) + @sliding_window.reject! { |val| val <= 3 } + assert_sliding_window(@sliding_window, [4, 5, 6, 7], 6) + end + + def test_sliding_window_reject_failure + @sliding_window << 0 << 1 << 2 << 3 << 4 << 5 << 6 << 7; + assert_equal(6, @sliding_window.size) + assert_sliding_window(@sliding_window, [2, 3, 4, 5, 6, 7], 6) + assert_raises ArgumentError do + # This deletes from the middle of the array + @sliding_window.reject! { |val| val == 3 } + end + end + def resize_to_less_than_1_raises assert_raises ArgumentError do @sliding_window.resize_to 0 @@ -33,9 +51,7 @@ def resize_to_less_than_1_raises private def assert_sliding_window(sliding_window, array, max_size) - # Get private member, the sliding_window doesn't expose the entire array - data = sliding_window.instance_variable_get("@window") - assert_equal(array, data) + assert_equal(array, sliding_window.values) assert_equal(max_size, sliding_window.max_size) end end From 3d78bbecd2d07abd16bd5de115f7932a2433d975 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Wed, 12 Jun 2019 16:45:29 -0400 Subject: [PATCH 05/49] Compile in C99 mode --- ext/semian/extconf.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/semian/extconf.rb b/ext/semian/extconf.rb index ac2c511b..ff000666 100644 --- a/ext/semian/extconf.rb +++ b/ext/semian/extconf.rb @@ -23,7 +23,7 @@ have_func 'rb_thread_blocking_region' have_func 'rb_thread_call_without_gvl' -$CFLAGS = "-D_GNU_SOURCE -Werror -Wall " +$CFLAGS = "-D_GNU_SOURCE -Werror -Wall -std=gnu99 " if ENV.key?('DEBUG') $CFLAGS << "-O0 -g -DDEBUG" else From e20c0781683a1b80d4999785dcc8e501655ee0be Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Wed, 12 Jun 2019 16:45:43 -0400 Subject: [PATCH 06/49] Fixes from code review --- ext/semian/simple_integer.c | 62 +++++++++++++++++------------ ext/semian/simple_integer.h | 4 +- ext/semian/sliding_window.c | 2 +- ext/semian/sysv_semaphores.c | 5 +-- ext/semian/types.h | 15 ++++--- lib/semian/circuit_breaker.rb | 4 -- lib/semian/simple_integer.rb | 5 +-- lib/semian/simple_sliding_window.rb | 6 +-- lib/semian/simple_state.rb | 6 +-- test/circuit_breaker_test.rb | 8 ---- test/redis_test.rb | 2 - test/resource_test.rb | 3 +- test/simple_integer_test.rb | 2 +- test/simple_sliding_window_test.rb | 4 +- 14 files changed, 61 insertions(+), 67 deletions(-) diff --git a/ext/semian/simple_integer.c b/ext/semian/simple_integer.c index 2966d247..c5f5a746 100644 --- a/ext/semian/simple_integer.c +++ b/ext/semian/simple_integer.c @@ -8,16 +8,12 @@ static const rb_data_type_t semian_simple_integer_type; -static int* get_value(VALUE self) { - const char* name = to_s(rb_iv_get(self, "@name")); - if (name == NULL) { - rb_raise(rb_eArgError, "could not get object name"); - } - - key_t key = generate_key(name); +static semian_simple_integer_shared_t* get_value(VALUE self) { + semian_simple_integer_t *res; + TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); const int permissions = 0664; - int shmid = shmget(key, sizeof(int), IPC_CREAT | permissions); + int shmid = shmget(res->key, sizeof(semian_simple_integer_shared_t), IPC_CREAT | permissions); if (shmid == -1) { rb_raise(rb_eArgError, "could not create shared memory (%s)", strerror(errno)); } @@ -27,7 +23,7 @@ static int* get_value(VALUE self) { rb_raise(rb_eArgError, "could not get shared memory (%s)", strerror(errno)); } - return (int*)val; + return (semian_simple_integer_shared_t*)val; } void Init_SimpleInteger() @@ -37,8 +33,8 @@ void Init_SimpleInteger() VALUE cSimpleInteger = rb_const_get(cSimple, rb_intern("Integer")); rb_define_alloc_func(cSimpleInteger, semian_simple_integer_alloc); - rb_define_method(cSimpleInteger, "initialize_simple_integer", semian_simple_integer_initialize, 0); - rb_define_method(cSimpleInteger, "increment", semian_simple_integer_increment, 1); + rb_define_method(cSimpleInteger, "initialize_simple_integer", semian_simple_integer_initialize, 1); + rb_define_method(cSimpleInteger, "increment", semian_simple_integer_increment, -1); rb_define_method(cSimpleInteger, "reset", semian_simple_integer_reset, 0); rb_define_method(cSimpleInteger, "value", semian_simple_integer_value_get, 0); rb_define_method(cSimpleInteger, "value=", semian_simple_integer_value_set, 1); @@ -52,39 +48,53 @@ VALUE semian_simple_integer_alloc(VALUE klass) } -VALUE semian_simple_integer_initialize(VALUE self) +VALUE semian_simple_integer_initialize(VALUE self, VALUE name) { - int* data = get_value(self); - *data = 0; + semian_simple_integer_t *res; + TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); + res->key = generate_key(to_s(name)); + + semian_simple_integer_shared_t* data = get_value(self); + data->val = 0; return self; } -VALUE semian_simple_integer_increment(VALUE self, VALUE val) { - int *data = get_value(self); - *data += rb_num2int(val); +VALUE semian_simple_integer_increment(int argc, VALUE *argv, VALUE self) { + // This is definitely the worst API ever. + // https://silverhammermba.github.io/emberb/c/#parsing-arguments + VALUE val; + rb_scan_args(argc, argv, "01", &val); + + semian_simple_integer_shared_t *data = get_value(self); + + if (NIL_P(val)) { + data->val += 1; + } else { + data->val += RB_NUM2INT(val); + } - return RB_INT2NUM(*data); + return RB_INT2NUM(data->val); } VALUE semian_simple_integer_reset(VALUE self) { - int *data = get_value(self); - *data = 0; + semian_simple_integer_shared_t *data = get_value(self); + data->val = 0; - return RB_INT2NUM(*data); + return RB_INT2NUM(data->val); } VALUE semian_simple_integer_value_get(VALUE self) { - int *data = get_value(self); - return RB_INT2NUM(*data); + semian_simple_integer_shared_t *data = get_value(self); + return RB_INT2NUM(data->val); } VALUE semian_simple_integer_value_set(VALUE self, VALUE val) { - int *data = get_value(self); + semian_simple_integer_shared_t *data = get_value(self); // TODO(michaelkipper): Check for respond_to?(:to_i) before calling. VALUE to_i = rb_funcall(val, rb_intern("to_i"), 0); - *data = RB_NUM2INT(to_i); + data->val = RB_NUM2INT(to_i); - return RB_INT2NUM(*data); + return RB_INT2NUM(data->val); } diff --git a/ext/semian/simple_integer.h b/ext/semian/simple_integer.h index 5a7f5ff0..e0463a58 100644 --- a/ext/semian/simple_integer.h +++ b/ext/semian/simple_integer.h @@ -6,8 +6,8 @@ void Init_SimpleInteger(); VALUE semian_simple_integer_alloc(VALUE klass); -VALUE semian_simple_integer_initialize(VALUE self); -VALUE semian_simple_integer_increment(VALUE self, VALUE val); +VALUE semian_simple_integer_initialize(VALUE self, VALUE name); +VALUE semian_simple_integer_increment(int argc, VALUE *argv, VALUE self); VALUE semian_simple_integer_reset(VALUE self); VALUE semian_simple_integer_value_get(VALUE self); VALUE semian_simple_integer_value_set(VALUE self, VALUE val); diff --git a/ext/semian/sliding_window.c b/ext/semian/sliding_window.c index 3615f1eb..d9c5244e 100644 --- a/ext/semian/sliding_window.c +++ b/ext/semian/sliding_window.c @@ -38,7 +38,7 @@ static int check_max_size_arg(VALUE max_size) if (retval <= 0) { rb_raise(rb_eArgError, "max_size must be greater than zero"); } else if (retval > SLIDING_WINDOW_MAX_SIZE) { - rb_raise(rb_eArgError, "max_size must be less than %d", SLIDING_WINDOW_MAX_SIZE); + rb_raise(rb_eArgError, "max_size cannot be greater than %d", SLIDING_WINDOW_MAX_SIZE); } return retval; diff --git a/ext/semian/sysv_semaphores.c b/ext/semian/sysv_semaphores.c index 755a4d47..6f4c12e3 100644 --- a/ext/semian/sysv_semaphores.c +++ b/ext/semian/sysv_semaphores.c @@ -29,6 +29,7 @@ raise_semian_syscall_error(const char *syscall, int error_num) void initialize_semaphore_set(semian_resource_t* res, const char* id_str, long permissions, int tickets, double quota) { + res->key = generate_key(id_str); res->strkey = (char*) malloc((2 /*for 0x*/+ sizeof(uint64_t) /*actual key*/+ 1 /*null*/) * sizeof(char)); sprintf(res->strkey, "0x%08x", (unsigned int) res->key); @@ -51,10 +52,6 @@ initialize_semaphore_set(semian_resource_t* res, const char* id_str, long permis } } -# if DEBUG - printf("[DEBUG] Init semaphore '%s' (key %s) to sem_id %d\n", res->name, res->strkey, res->sem_id); -# endif - set_semaphore_permissions(res->sem_id, permissions); /* diff --git a/ext/semian/types.h b/ext/semian/types.h index 70ec5f7a..1ba58fa5 100644 --- a/ext/semian/types.h +++ b/ext/semian/types.h @@ -58,6 +58,16 @@ typedef struct { uint64_t key; } semian_simple_integer_t; +// Shared simple integer structure +typedef struct { + int val; +} semian_simple_integer_shared_t; + +// Internal simple sliding window structure +typedef struct { + uint64_t key; +} semian_simple_sliding_window_t; + // Shared simple sliding window structure typedef struct { int max_size; @@ -67,9 +77,4 @@ typedef struct { int data[SLIDING_WINDOW_MAX_SIZE]; } semian_simple_sliding_window_shared_t; -// Internal simple sliding window structure -typedef struct { - uint64_t key; -} semian_simple_sliding_window_t; - #endif // SEMIAN_TYPES_H diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index 0dccbe88..8225026c 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -26,8 +26,6 @@ def initialize(name, exceptions:, success_threshold:, error_threshold:, end def acquire(resource = nil, &block) - puts "[DEBUG] #{Time.now} - Acquiring resource '#{resource}'" - return yield if disabled? transition_to_half_open if transition_to_half_open? @@ -119,7 +117,6 @@ def error_threshold_reached? end def error_timeout_expired? - puts "[DEBUG] Checking error_timeout_expired? #{@errors.last}" last_error_time = @errors.last return false unless last_error_time Time.at(last_error_time) + @error_timeout < Time.now @@ -130,7 +127,6 @@ def push_error(error) end def push_time(window, time: Time.now) - puts "[DEBUG] push_time(#{time.to_i}) - rejecting before #{time.to_i - @error_timeout}" window.reject! { |err_time| err_time + @error_timeout < time.to_i } window << time.to_i end diff --git a/lib/semian/simple_integer.rb b/lib/semian/simple_integer.rb index 3b912fbf..e29185d8 100644 --- a/lib/semian/simple_integer.rb +++ b/lib/semian/simple_integer.rb @@ -6,12 +6,11 @@ class Integer #:nodoc: attr_accessor :value def initialize(name) - @name = name - initialize_simple_integer if respond_to?(:initialize_simple_integer) + initialize_simple_integer(name) if respond_to?(:initialize_simple_integer) reset end - def increment(val) + def increment(val = 1) @value += val end diff --git a/lib/semian/simple_sliding_window.rb b/lib/semian/simple_sliding_window.rb index 15ad1e43..bd0c2ed8 100644 --- a/lib/semian/simple_sliding_window.rb +++ b/lib/semian/simple_sliding_window.rb @@ -3,6 +3,8 @@ module Semian module Simple class SlidingWindow #:nodoc: + attr_reader :max_size + # A sliding window is a structure that stores the most @max_size recent timestamps # like this: if @max_size = 4, current time is 10, @window =[5,7,9,10]. # Another push of (11) at 11 sec would make @window [7,9,10,11], shifting off 5. @@ -17,10 +19,6 @@ def size @window.size end - def max_size - @max_size - end - def last @window.last end diff --git a/lib/semian/simple_state.rb b/lib/semian/simple_state.rb index 228c2e41..0357a345 100644 --- a/lib/semian/simple_state.rb +++ b/lib/semian/simple_state.rb @@ -6,9 +6,9 @@ class State #:nodoc: def_delegators :@value, :value # State constants. Looks like a flag, but is actually an enum. - UNKNOWN = 0x0 - CLOSED = 0x1 - OPEN = 0x2 + UNKNOWN = 0x0 + CLOSED = 0x1 + OPEN = 0x2 HALF_OPEN = 0x4 def initialize(value) diff --git a/test/circuit_breaker_test.rb b/test/circuit_breaker_test.rb index 594d4a07..be4c01c9 100644 --- a/test/circuit_breaker_test.rb +++ b/test/circuit_breaker_test.rb @@ -11,8 +11,6 @@ def setup rescue nil end - - puts "[DEBUG] Registering semian :testing at #{Time.now} (#{Time.now.to_i})" Semian.register(:testing, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) @resource = Semian[:testing] end @@ -42,14 +40,8 @@ def test_after_error_timeout_is_elapsed_requests_are_attempted_again end def test_until_success_threshold_is_reached_a_single_error_will_reopen_the_circuit - puts @strio.string - puts "[DEBUG] Half open circuit..." half_open_cicuit! - puts @strio.string - puts "[DEBUG] Trigger error..." trigger_error! - puts @strio.string - puts "[DEBUG] Assert circuit opened..." assert_circuit_opened assert_match(/State transition from open to half_open/, @strio.string) end diff --git a/test/redis_test.rb b/test/redis_test.rb index c13e5551..1ae2dbaf 100644 --- a/test/redis_test.rb +++ b/test/redis_test.rb @@ -288,9 +288,7 @@ def test_circuit_breaker_on_query client.get('foo') end - puts "[DEBUG] #1 Time.now = #{Time.now}" Timecop.travel(ERROR_TIMEOUT + 1) do - puts "[DEBUG] #2 Time.now = #{Time.now}" assert_equal '2', client.get('foo') end end diff --git a/test/resource_test.rb b/test/resource_test.rb index 95487df0..aa624a42 100644 --- a/test/resource_test.rb +++ b/test/resource_test.rb @@ -1,5 +1,4 @@ require 'test_helper' - require 'objspace' class TestResource < Minitest::Test @@ -494,7 +493,7 @@ def test_multiple_register_with_fork def test_memsize r = create_resource :testing, tickets: 1 - puts "mkipper: Resource size is #{ObjectSpace.memsize_of(r)} byte(s)" + assert_equal 128, ObjectSpace.memsize_of(r) end def create_resource(*args) diff --git a/test/simple_integer_test.rb b/test/simple_integer_test.rb index aa03eeb1..ea7e1697 100644 --- a/test/simple_integer_test.rb +++ b/test/simple_integer_test.rb @@ -26,7 +26,7 @@ def test_access_value def test_increment @integer.increment(4) assert_equal(4, @integer.value) - @integer.increment(1) + @integer.increment assert_equal(5, @integer.value) @integer.increment(-2) assert_equal(3, @integer.value) diff --git a/test/simple_sliding_window_test.rb b/test/simple_sliding_window_test.rb index 618be2e5..d178e922 100644 --- a/test/simple_sliding_window_test.rb +++ b/test/simple_sliding_window_test.rb @@ -25,7 +25,7 @@ def test_sliding_window_edge_falloff end def test_sliding_window_reject - @sliding_window << 0 << 1 << 2 << 3 << 4 << 5 << 6 << 7; + @sliding_window << 0 << 1 << 2 << 3 << 4 << 5 << 6 << 7 assert_equal(6, @sliding_window.size) assert_sliding_window(@sliding_window, [2, 3, 4, 5, 6, 7], 6) @sliding_window.reject! { |val| val <= 3 } @@ -33,7 +33,7 @@ def test_sliding_window_reject end def test_sliding_window_reject_failure - @sliding_window << 0 << 1 << 2 << 3 << 4 << 5 << 6 << 7; + @sliding_window << 0 << 1 << 2 << 3 << 4 << 5 << 6 << 7 assert_equal(6, @sliding_window.size) assert_sliding_window(@sliding_window, [2, 3, 4, 5, 6, 7], 6) assert_raises ArgumentError do From 569a551e14e04a81149d0b1af97bd697800173e0 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Thu, 13 Jun 2019 16:41:08 -0400 Subject: [PATCH 07/49] Added a dprintf function --- ext/semian/circuit_breaker.c | 12 ++++++------ ext/semian/sliding_window.c | 34 +++++++++++++++++----------------- ext/semian/util.h | 16 ++++++++++++++++ 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/ext/semian/circuit_breaker.c b/ext/semian/circuit_breaker.c index 0f7f6f83..6bd3761b 100644 --- a/ext/semian/circuit_breaker.c +++ b/ext/semian/circuit_breaker.c @@ -9,7 +9,7 @@ static const rb_data_type_t semian_circuit_breaker_type; void Init_CircuitBreaker() { - printf("[DEBUG] Init_CircuitBreaker\n"); + dprintf("Init_CircuitBreaker"); VALUE cSemian = rb_const_get(rb_cObject, rb_intern("Semian")); VALUE cCircuitBreaker = rb_const_get(cSemian, rb_intern("CircuitBreaker")); @@ -20,7 +20,7 @@ void Init_CircuitBreaker() { VALUE semian_circuit_breaker_alloc(VALUE klass) { - printf("[DEBUG] semian_circuit_breaker_alloc\n"); + dprintf("semian_circuit_breaker_alloc"); semian_circuit_breaker_t *res; VALUE obj = TypedData_Make_Struct(klass, semian_circuit_breaker_t, &semian_circuit_breaker_type, res); @@ -31,7 +31,7 @@ VALUE semian_circuit_breaker_initialize(VALUE self, VALUE id) { const char *c_id_str = check_id_arg(id); - printf("[DEBUG] semian_circuit_breaker_initialize('%s')\n", c_id_str); + dprintf("semian_circuit_breaker_initialize('%s')", c_id_str); // Build semian resource structure semian_circuit_breaker_t *res = NULL; @@ -42,14 +42,14 @@ VALUE semian_circuit_breaker_initialize(VALUE self, VALUE id) res->name = strdup(c_id_str); key_t key = generate_key(res->name); - printf("[DEBUG] Creating shared memory for '%s' (id %u)\n", res->name, key); + dprintf("Creating shared memory for '%s' (id %u)", res->name, key); const int permissions = 0664; int shmid = shmget(key, 1024, IPC_CREAT | permissions); if (shmid == -1) { rb_raise(rb_eArgError, "could not create shared memory (%s)", strerror(errno)); } - printf("[DEBUG] Getting shared memory (id %u)\n", shmid); + dprintf("Getting shared memory (id %u)", shmid); void *val = shmat(shmid, NULL, 0); if (val == (void*)-1) { rb_raise(rb_eArgError, "could not get shared memory (%s)", strerror(errno)); @@ -60,7 +60,7 @@ VALUE semian_circuit_breaker_initialize(VALUE self, VALUE id) rb_raise(rb_eArgError, "could not get shared memory (%s)", strerror(errno)); } - printf("[DEBUG] successes = %d\n", data->successes); + dprintf("successes = %d", data->successes); data->successes = 0; return self; diff --git a/ext/semian/sliding_window.c b/ext/semian/sliding_window.c index d9c5244e..1f370394 100644 --- a/ext/semian/sliding_window.c +++ b/ext/semian/sliding_window.c @@ -45,7 +45,7 @@ static int check_max_size_arg(VALUE max_size) } void Init_SlidingWindow() { - printf("[DEBUG] Init_SlidingWindow\n"); + dprintf("Init_SlidingWindow"); VALUE cSemian = rb_const_get(rb_cObject, rb_intern("Semian")); VALUE cSimple = rb_const_get(cSemian, rb_intern("Simple")); @@ -69,7 +69,7 @@ VALUE semian_simple_sliding_window_alloc(VALUE klass) { } VALUE semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_size) { - printf("[DEBUG] semian_simple_sliding_window_initialize\n"); + dprintf("semian_simple_sliding_window_initialize"); semian_simple_sliding_window_t *res; TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); @@ -83,37 +83,37 @@ VALUE semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_ window->start = 0; window->end = 0; - printf("[DEBUG] key:%lu addr:0x%p max_size:%d length:%d start:%d end:%d\n", res->key, window, window->max_size, window->length, window->start, window->end); + dprintf(" key:%lu addr:0x%p max_size:%d length:%d start:%d end:%d", res->key, window, window->max_size, window->length, window->start, window->end); return self; } VALUE semian_simple_sliding_window_size(VALUE self) { - printf("[DEBUG] semian_simple_sliding_window_size\n"); + dprintf("semian_simple_sliding_window_size"); semian_simple_sliding_window_t *res; TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); semian_simple_sliding_window_shared_t *window = get_window(res->key); - printf("[DEBUG] key:%lu addr:0x%p max_size:%d length:%d start:%d end:%d\n", res->key, window, window->max_size, window->length, window->start, window->end); + dprintf(" key:%lu addr:0x%p max_size:%d length:%d start:%d end:%d", res->key, window, window->max_size, window->length, window->start, window->end); return RB_INT2NUM(window->length); } VALUE semian_simple_sliding_window_max_size(VALUE self) { - printf("[DEBUG] semian_simple_sliding_window_max_size\n"); + dprintf("semian_simple_sliding_window_max_size"); semian_simple_sliding_window_t *res; TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); semian_simple_sliding_window_shared_t *window = get_window(res->key); - printf("[DEBUG] key:%lu addr:0x%p max_size:%d length:%d start:%d end:%d\n", res->key, window, window->max_size, window->length, window->start, window->end); + dprintf(" key:%lu addr:0x%p max_size:%d length:%d start:%d end:%d", res->key, window, window->max_size, window->length, window->start, window->end); return RB_INT2NUM(window->max_size); } VALUE semian_simple_sliding_window_values(VALUE self) { - printf("[DEBUG] semian_simple_sliding_window_values\n"); + dprintf("semian_simple_sliding_window_values"); semian_simple_sliding_window_t *res; TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); @@ -123,7 +123,7 @@ VALUE semian_simple_sliding_window_values(VALUE self) { for (int i = 0; i < window->length; ++i) { int index = (window->start + i) % window->max_size; int value = window->data[index]; - printf("[DEBUG] i:%d index: %d value:%d max_size:%d length:%d start:%d end:%d\n", i, index, value, window->max_size, window->length, window->start, window->end); + dprintf(" i:%d index: %d value:%d max_size:%d length:%d start:%d end:%d", i, index, value, window->max_size, window->length, window->start, window->end); rb_ary_store(retval, i, RB_INT2NUM(value)); } @@ -131,20 +131,20 @@ VALUE semian_simple_sliding_window_values(VALUE self) { } VALUE semian_simple_sliding_window_last(VALUE self) { - printf("[DEBUG] semian_simple_sliding_window_last\n"); + dprintf("semian_simple_sliding_window_last"); semian_simple_sliding_window_t *res; TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); semian_simple_sliding_window_shared_t *window = get_window(res->key); int index = (window->start + window->length - 1) % window->max_size; - printf("[DEBUG] index:%d last:%d\n", index, window->data[index]); + dprintf(" index:%d last:%d", index, window->data[index]); return RB_INT2NUM(window->data[index]); } VALUE semian_simple_sliding_window_clear(VALUE self) { - printf("[DEBUG] semian_simple_sliding_window_clear\n"); + dprintf("semian_simple_sliding_window_clear"); semian_simple_sliding_window_t *res; TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); @@ -158,7 +158,7 @@ VALUE semian_simple_sliding_window_clear(VALUE self) { } VALUE semian_simple_sliding_window_reject(VALUE self) { - printf("[DEBUG] semian_simple_sliding_window_reject\n"); + dprintf("semian_simple_sliding_window_reject"); rb_need_block(); @@ -174,7 +174,7 @@ VALUE semian_simple_sliding_window_reject(VALUE self) { for (int i = 0; i < length; ++i) { int index = (start + i) % length; int value = window->data[index]; - printf("[DEBUG] i:%d index: %d value:%d max_size:%d length:%d start:%d end:%d\n", i, index, value, window->max_size, window->length, window->start, window->end); + dprintf(" i:%d index: %d value:%d max_size:%d length:%d start:%d end:%d", i, index, value, window->max_size, window->length, window->start, window->end); VALUE y = rb_yield(RB_INT2NUM(value)); if (RTEST(y)) { if (cleared++ != i) { @@ -182,7 +182,7 @@ VALUE semian_simple_sliding_window_reject(VALUE self) { } window->start = (window->start + 1) % window->length; window->length--; - printf("[DEBUG] Removed index:%d (val:%d)\n", i, value); + dprintf(" Removed index:%d (val:%d)", i, value); } } @@ -190,7 +190,7 @@ VALUE semian_simple_sliding_window_reject(VALUE self) { } VALUE semian_simple_sliding_window_push(VALUE self, VALUE value) { - printf("[DEBUG] semian_simple_sliding_window_push\n"); + dprintf("semian_simple_sliding_window_push"); semian_simple_sliding_window_t *res; TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); @@ -206,7 +206,7 @@ VALUE semian_simple_sliding_window_push(VALUE self, VALUE value) { window->data[index] = RB_NUM2INT(value); window->end = (window->end + 1) % window->max_size; - printf("[DEBUG] Pushed val:%d index:%d max_size:%d length:%d start:%d end:%d\n", RB_NUM2INT(value), index, window->max_size, window->length, window->start, window->end); + dprintf(" Pushed val:%d index:%d max_size:%d length:%d start:%d end:%d", RB_NUM2INT(value), index, window->max_size, window->length, window->start, window->end); return self; } diff --git a/ext/semian/util.h b/ext/semian/util.h index f439e4c3..670b7d1e 100644 --- a/ext/semian/util.h +++ b/ext/semian/util.h @@ -1,9 +1,25 @@ #ifndef EXT_SEMIAN_UTIL_H #define EXT_SEMIAN_UTIL_H +#include +#include + #include #include +#ifdef DEBUG +# define DEBUG_TEST 1 +#else +# define DEBUG_TEST 0 +#endif + +#define dprintf(fmt, ...) \ + do { \ + if (DEBUG_TEST) { \ + printf("[DEBUG] %s:%d" fmt "\n", __FILE__, __LINE__, ##__VA_ARGS__); \ + } \ + } while (0) + const char* check_id_arg(VALUE id); key_t generate_key(const char *name); From f55e521f3d4fc37dd15d59ff1ef0a9be3bd0c9b1 Mon Sep 17 00:00:00 2001 From: Spike Lindsey Date: Thu, 13 Jun 2019 10:13:00 -0400 Subject: [PATCH 08/49] Write failing race test --- ext/semian/simple_integer.c | 2 ++ test/simple_integer_test.rb | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/ext/semian/simple_integer.c b/ext/semian/simple_integer.c index c5f5a746..47d6fea2 100644 --- a/ext/semian/simple_integer.c +++ b/ext/semian/simple_integer.c @@ -28,6 +28,8 @@ static semian_simple_integer_shared_t* get_value(VALUE self) { void Init_SimpleInteger() { + printf("[DEBUG] Init_SimpleInteger\n"); + VALUE cSemian = rb_const_get(rb_cObject, rb_intern("Semian")); VALUE cSimple = rb_const_get(cSemian, rb_intern("Simple")); VALUE cSimpleInteger = rb_const_get(cSimple, rb_intern("Integer")); diff --git a/test/simple_integer_test.rb b/test/simple_integer_test.rb index ea7e1697..5b196896 100644 --- a/test/simple_integer_test.rb +++ b/test/simple_integer_test.rb @@ -41,6 +41,28 @@ def test_reset @integer.reset assert_equal(0, @integer.value) end + + def test_integer_race + process_count = 255 + + 5.times do + process_count.times do + fork do + # An attempt to run all the increments at approximately the same + # time + loop until Time.now.sec.modulo(2).zero? + value = @integer.increment(1) + exit!(value) + end + end + + finished_processes = Process.waitall + + exit_codes = finished_processes.map { |_, status| status.exitstatus } + assert_equal((1..process_count).to_a, exit_codes) + @integer.reset + end + end end include IntegerTestCases From f517f7504a32ebd73580bd10d0538622adea2bf8 Mon Sep 17 00:00:00 2001 From: Spike Lindsey Date: Thu, 13 Jun 2019 10:18:25 -0400 Subject: [PATCH 09/49] Allow fallback to Ruby to be toggled by env var --- ext/semian/semian.c | 8 +++++--- test/simple_integer_test.rb | 32 +++++++++++++++++--------------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/ext/semian/semian.c b/ext/semian/semian.c index 1b730473..94910f94 100644 --- a/ext/semian/semian.c +++ b/ext/semian/semian.c @@ -65,7 +65,9 @@ void Init_semian() /* Maximum number of tickets available on this system. */ rb_define_const(cSemian, "MAX_TICKETS", INT2FIX(system_max_semaphore_count)); - Init_SimpleInteger(); - Init_SlidingWindow(); - Init_CircuitBreaker(); + if (getenv("USE_RUBY_CIRCUITS") == NULL || strcmp(getenv("USE_RUBY_CIRCUITS"), "false")) { + Init_SimpleInteger(); + Init_SlidingWindow(); + Init_CircuitBreaker(); + } } diff --git a/test/simple_integer_test.rb b/test/simple_integer_test.rb index 5b196896..aa0343d8 100644 --- a/test/simple_integer_test.rb +++ b/test/simple_integer_test.rb @@ -42,25 +42,27 @@ def test_reset assert_equal(0, @integer.value) end - def test_integer_race - process_count = 255 + unless ENV['USE_RUBY_CIRCUITS'] + def test_integer_race + process_count = 255 - 5.times do - process_count.times do - fork do - # An attempt to run all the increments at approximately the same - # time - loop until Time.now.sec.modulo(2).zero? - value = @integer.increment(1) - exit!(value) + 5.times do + process_count.times do + fork do + # An attempt to run all the increments at approximately the same + # time + loop until Time.now.sec.modulo(2).zero? + value = @integer.increment(1) + exit!(value) + end end - end - finished_processes = Process.waitall + finished_processes = Process.waitall - exit_codes = finished_processes.map { |_, status| status.exitstatus } - assert_equal((1..process_count).to_a, exit_codes) - @integer.reset + exit_codes = finished_processes.map { |_, status| status.exitstatus } + assert_equal((1..process_count).to_a, exit_codes) + @integer.reset + end end end end From 7c0fc801b9d566ac6aded99f57b8b82bf8ef5d3c Mon Sep 17 00:00:00 2001 From: Spike Lindsey Date: Thu, 13 Jun 2019 10:46:43 -0400 Subject: [PATCH 10/49] Set up single semaphore --- ext/semian/simple_integer.c | 60 ++++++++++++++++++++++++++++-------- ext/semian/sysv_semaphores.c | 36 ++++++++++++++++++++++ ext/semian/sysv_semaphores.h | 5 +++ ext/semian/types.h | 1 + test/simple_integer_test.rb | 20 +++++------- 5 files changed, 97 insertions(+), 25 deletions(-) diff --git a/ext/semian/simple_integer.c b/ext/semian/simple_integer.c index 47d6fea2..9428ded6 100644 --- a/ext/semian/simple_integer.c +++ b/ext/semian/simple_integer.c @@ -5,13 +5,11 @@ #include #include "types.h" #include "util.h" +#include "sysv_semaphores.h" static const rb_data_type_t semian_simple_integer_type; -static semian_simple_integer_shared_t* get_value(VALUE self) { - semian_simple_integer_t *res; - TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); - +static semian_simple_integer_shared_t* get_value(semian_simple_integer_t* res) { const int permissions = 0664; int shmid = shmget(res->key, sizeof(semian_simple_integer_shared_t), IPC_CREAT | permissions); if (shmid == -1) { @@ -56,9 +54,12 @@ VALUE semian_simple_integer_initialize(VALUE self, VALUE name) TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); res->key = generate_key(to_s(name)); - semian_simple_integer_shared_t* data = get_value(self); + semian_simple_integer_shared_t* data = get_value(res); data->val = 0; + const int permissions = 0664; + res->sem_id = initialize_single_semaphore(res->key, permissions); + return self; } @@ -68,35 +69,68 @@ VALUE semian_simple_integer_increment(int argc, VALUE *argv, VALUE self) { VALUE val; rb_scan_args(argc, argv, "01", &val); - semian_simple_integer_shared_t *data = get_value(self); + semian_simple_integer_t *res; + TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); + + sem_meta_lock(res->sem_id); + + semian_simple_integer_shared_t *data = get_value(res); if (NIL_P(val)) { data->val += 1; } else { data->val += RB_NUM2INT(val); } + VALUE rb_val = RB_INT2NUM(data->val); + + sem_meta_unlock(res->sem_id); - return RB_INT2NUM(data->val); + return rb_val; } VALUE semian_simple_integer_reset(VALUE self) { - semian_simple_integer_shared_t *data = get_value(self); + semian_simple_integer_t *res; + TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); + + sem_meta_lock(res->sem_id); + + semian_simple_integer_shared_t *data = get_value(res); data->val = 0; + VALUE rb_val = RB_INT2NUM(data->val); + + sem_meta_unlock(res->sem_id); - return RB_INT2NUM(data->val); + return rb_val; } VALUE semian_simple_integer_value_get(VALUE self) { - semian_simple_integer_shared_t *data = get_value(self); - return RB_INT2NUM(data->val); + semian_simple_integer_t *res; + TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); + + sem_meta_lock(res->sem_id); + + semian_simple_integer_shared_t *data = get_value(res); + VALUE rb_val = RB_INT2NUM(data->val); + + sem_meta_unlock(res->sem_id); + + return rb_val; } VALUE semian_simple_integer_value_set(VALUE self, VALUE val) { - semian_simple_integer_shared_t *data = get_value(self); + semian_simple_integer_t *res; + TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); + + sem_meta_lock(res->sem_id); + + semian_simple_integer_shared_t *data = get_value(res); // TODO(michaelkipper): Check for respond_to?(:to_i) before calling. VALUE to_i = rb_funcall(val, rb_intern("to_i"), 0); data->val = RB_NUM2INT(to_i); + VALUE rb_val = RB_INT2NUM(data->val); + + sem_meta_unlock(res->sem_id); - return RB_INT2NUM(data->val); + return rb_val; } diff --git a/ext/semian/sysv_semaphores.c b/ext/semian/sysv_semaphores.c index 6f4c12e3..b469413e 100644 --- a/ext/semian/sysv_semaphores.c +++ b/ext/semian/sysv_semaphores.c @@ -33,6 +33,7 @@ initialize_semaphore_set(semian_resource_t* res, const char* id_str, long permis res->key = generate_key(id_str); res->strkey = (char*) malloc((2 /*for 0x*/+ sizeof(uint64_t) /*actual key*/+ 1 /*null*/) * sizeof(char)); sprintf(res->strkey, "0x%08x", (unsigned int) res->key); + res->sem_id = semget(res->key, SI_NUM_SEMAPHORES, IPC_CREAT | IPC_EXCL | permissions); /* @@ -243,3 +244,38 @@ diff_timespec_ms(struct timespec *end, struct timespec *begin) long begin_ms = (begin->tv_sec * 1e3) + (begin->tv_nsec / 1e6); return end_ms - begin_ms; } + +int +initialize_single_semaphore(key_t key, long permissions) +{ + + int sem_id = semget(key, 1, IPC_CREAT | IPC_EXCL | permissions); + + /* + This approach is based on http://man7.org/tlpi/code/online/dist/svsem/svsem_good_init.c.html + which avoids race conditions when initializing semaphore sets. + */ + if (sem_id != -1) { + // Happy path - we are the first worker, initialize the semaphore set. + // + if (semctl(sem_id, 0, SETVAL, 1) == -1) { + raise_semian_syscall_error("semctl()", errno); + } + } else { + // Something went wrong + if (errno != EEXIST) { + raise_semian_syscall_error("semget() failed to initialize semaphore values", errno); + } else { + // The semaphore set already exists, ensure it is initialized + sem_id = wait_for_new_semaphore_set(key, permissions); + } + } + + set_semaphore_permissions(sem_id, permissions); + + sem_meta_lock(sem_id); // Sets otime for the first time by acquiring the sem lock + + sem_meta_unlock(sem_id); + + return sem_id; +} diff --git a/ext/semian/sysv_semaphores.h b/ext/semian/sysv_semaphores.h index 205a6cbb..e93ca4a2 100644 --- a/ext/semian/sysv_semaphores.h +++ b/ext/semian/sysv_semaphores.h @@ -105,6 +105,11 @@ get_semaphore(int key); void * acquire_semaphore_without_gvl(void *p); +// Initializes a semaphore set with a single semaphore, for general purpose +// locking +int +initialize_single_semaphore(key_t key, long permissions); + #ifdef DEBUG static inline void print_sem_vals(int sem_id) diff --git a/ext/semian/types.h b/ext/semian/types.h index 1ba58fa5..68b94e6b 100644 --- a/ext/semian/types.h +++ b/ext/semian/types.h @@ -56,6 +56,7 @@ typedef struct { // Internal simple integer structure typedef struct { uint64_t key; + int sem_id; } semian_simple_integer_t; // Shared simple integer structure diff --git a/test/simple_integer_test.rb b/test/simple_integer_test.rb index aa0343d8..4e8b76e4 100644 --- a/test/simple_integer_test.rb +++ b/test/simple_integer_test.rb @@ -43,25 +43,21 @@ def test_reset end unless ENV['USE_RUBY_CIRCUITS'] - def test_integer_race + # Without locks, this only passes around 1 in every 5 runs + def test_increment_race process_count = 255 - - 5.times do + 100.times do process_count.times do fork do - # An attempt to run all the increments at approximately the same - # time - loop until Time.now.sec.modulo(2).zero? value = @integer.increment(1) exit!(value) end end - - finished_processes = Process.waitall - - exit_codes = finished_processes.map { |_, status| status.exitstatus } - assert_equal((1..process_count).to_a, exit_codes) - @integer.reset + exit_codes = Process.waitall.map { |_, status| status.exitstatus } + # No two processes should exit with the same exit code + duplicate_values = exit_codes.group_by { |i| i }.select { |_, v| v.size > 1 } + puts "Duplicate values: #{duplicate_values}" unless duplicate_values.empty? + assert_equal(process_count, exit_codes.uniq.length) end end end From 7325b4734b58bcda4e4d3358008fd16f653c06dd Mon Sep 17 00:00:00 2001 From: Spike Lindsey Date: Thu, 13 Jun 2019 17:14:28 -0400 Subject: [PATCH 11/49] Move default permissions into sysv_semaphore header --- ext/semian/simple_integer.c | 6 ++---- ext/semian/sysv_semaphores.c | 1 - ext/semian/sysv_semaphores.h | 4 ++++ 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ext/semian/simple_integer.c b/ext/semian/simple_integer.c index 9428ded6..5d481f8e 100644 --- a/ext/semian/simple_integer.c +++ b/ext/semian/simple_integer.c @@ -10,8 +10,7 @@ static const rb_data_type_t semian_simple_integer_type; static semian_simple_integer_shared_t* get_value(semian_simple_integer_t* res) { - const int permissions = 0664; - int shmid = shmget(res->key, sizeof(semian_simple_integer_shared_t), IPC_CREAT | permissions); + int shmid = shmget(res->key, sizeof(semian_simple_integer_shared_t), IPC_CREAT | SEM_DEFAULT_PERMISSIONS); if (shmid == -1) { rb_raise(rb_eArgError, "could not create shared memory (%s)", strerror(errno)); } @@ -57,8 +56,7 @@ VALUE semian_simple_integer_initialize(VALUE self, VALUE name) semian_simple_integer_shared_t* data = get_value(res); data->val = 0; - const int permissions = 0664; - res->sem_id = initialize_single_semaphore(res->key, permissions); + res->sem_id = initialize_single_semaphore(res->key, SEM_DEFAULT_PERMISSIONS); return self; } diff --git a/ext/semian/sysv_semaphores.c b/ext/semian/sysv_semaphores.c index b469413e..79c884ce 100644 --- a/ext/semian/sysv_semaphores.c +++ b/ext/semian/sysv_semaphores.c @@ -248,7 +248,6 @@ diff_timespec_ms(struct timespec *end, struct timespec *begin) int initialize_single_semaphore(key_t key, long permissions) { - int sem_id = semget(key, 1, IPC_CREAT | IPC_EXCL | permissions); /* diff --git a/ext/semian/sysv_semaphores.h b/ext/semian/sysv_semaphores.h index e93ca4a2..d0890c8c 100644 --- a/ext/semian/sysv_semaphores.h +++ b/ext/semian/sysv_semaphores.h @@ -38,6 +38,10 @@ typedef VALUE (*my_blocking_fn_t)(void*); // Helper definition to prevent magic number for conversion of microseconds to seconds #define MICROSECONDS_IN_SECOND 1000000 +// Default permissions for semaphore set - execute permissions are not meaningful for +// semaphores +#define SEM_DEFAULT_PERMISSIONS 0660 + // Here we define an enum value and string representation of each semaphore // This allows us to key the sem value and string rep in sync easily // utilizing pre-processor macros. From 3c45b3f5f993eef90e49f0a84fa02c7d81a8cc79 Mon Sep 17 00:00:00 2001 From: Spike Lindsey Date: Thu, 13 Jun 2019 17:14:53 -0400 Subject: [PATCH 12/49] Clarify critical sections --- ext/semian/simple_integer.c | 60 +++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/ext/semian/simple_integer.c b/ext/semian/simple_integer.c index 5d481f8e..37c9aec7 100644 --- a/ext/semian/simple_integer.c +++ b/ext/semian/simple_integer.c @@ -65,22 +65,24 @@ VALUE semian_simple_integer_increment(int argc, VALUE *argv, VALUE self) { // This is definitely the worst API ever. // https://silverhammermba.github.io/emberb/c/#parsing-arguments VALUE val; + semian_simple_integer_t *res; + VALUE rb_val; + rb_scan_args(argc, argv, "01", &val); - semian_simple_integer_t *res; TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); sem_meta_lock(res->sem_id); - - semian_simple_integer_shared_t *data = get_value(res); - - if (NIL_P(val)) { - data->val += 1; - } else { - data->val += RB_NUM2INT(val); + { + semian_simple_integer_shared_t *data = get_value(res); + + if (NIL_P(val)) { + data->val += 1; + } else { + data->val += RB_NUM2INT(val); + } + rb_val = RB_INT2NUM(data->val); } - VALUE rb_val = RB_INT2NUM(data->val); - sem_meta_unlock(res->sem_id); return rb_val; @@ -88,14 +90,16 @@ VALUE semian_simple_integer_increment(int argc, VALUE *argv, VALUE self) { VALUE semian_simple_integer_reset(VALUE self) { semian_simple_integer_t *res; + VALUE rb_val; + TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); sem_meta_lock(res->sem_id); - - semian_simple_integer_shared_t *data = get_value(res); - data->val = 0; - VALUE rb_val = RB_INT2NUM(data->val); - + { + semian_simple_integer_shared_t *data = get_value(res); + data->val = 0; + rb_val = RB_INT2NUM(data->val); + } sem_meta_unlock(res->sem_id); return rb_val; @@ -103,13 +107,15 @@ VALUE semian_simple_integer_reset(VALUE self) { VALUE semian_simple_integer_value_get(VALUE self) { semian_simple_integer_t *res; + VALUE rb_val; + TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); sem_meta_lock(res->sem_id); - - semian_simple_integer_shared_t *data = get_value(res); - VALUE rb_val = RB_INT2NUM(data->val); - + { + semian_simple_integer_shared_t *data = get_value(res); + rb_val = RB_INT2NUM(data->val); + } sem_meta_unlock(res->sem_id); return rb_val; @@ -117,17 +123,19 @@ VALUE semian_simple_integer_value_get(VALUE self) { VALUE semian_simple_integer_value_set(VALUE self, VALUE val) { semian_simple_integer_t *res; + VALUE rb_val; + TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); sem_meta_lock(res->sem_id); + { + semian_simple_integer_shared_t *data = get_value(res); - semian_simple_integer_shared_t *data = get_value(res); - - // TODO(michaelkipper): Check for respond_to?(:to_i) before calling. - VALUE to_i = rb_funcall(val, rb_intern("to_i"), 0); - data->val = RB_NUM2INT(to_i); - VALUE rb_val = RB_INT2NUM(data->val); - + // TODO(michaelkipper): Check for respond_to?(:to_i) before calling. + VALUE to_i = rb_funcall(val, rb_intern("to_i"), 0); + data->val = RB_NUM2INT(to_i); + rb_val = RB_INT2NUM(data->val); + } sem_meta_unlock(res->sem_id); return rb_val; From 35217ae14afac467d871afd3787743228f8c173b Mon Sep 17 00:00:00 2001 From: Spike Lindsey Date: Fri, 14 Jun 2019 10:44:05 -0400 Subject: [PATCH 13/49] Rename rb_val->retval --- ext/semian/simple_integer.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/semian/simple_integer.c b/ext/semian/simple_integer.c index 37c9aec7..94676ca2 100644 --- a/ext/semian/simple_integer.c +++ b/ext/semian/simple_integer.c @@ -66,7 +66,7 @@ VALUE semian_simple_integer_increment(int argc, VALUE *argv, VALUE self) { // https://silverhammermba.github.io/emberb/c/#parsing-arguments VALUE val; semian_simple_integer_t *res; - VALUE rb_val; + VALUE retval; rb_scan_args(argc, argv, "01", &val); @@ -81,16 +81,16 @@ VALUE semian_simple_integer_increment(int argc, VALUE *argv, VALUE self) { } else { data->val += RB_NUM2INT(val); } - rb_val = RB_INT2NUM(data->val); + retval = RB_INT2NUM(data->val); } sem_meta_unlock(res->sem_id); - return rb_val; + return retval; } VALUE semian_simple_integer_reset(VALUE self) { semian_simple_integer_t *res; - VALUE rb_val; + VALUE retval; TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); @@ -98,32 +98,32 @@ VALUE semian_simple_integer_reset(VALUE self) { { semian_simple_integer_shared_t *data = get_value(res); data->val = 0; - rb_val = RB_INT2NUM(data->val); + retval = RB_INT2NUM(data->val); } sem_meta_unlock(res->sem_id); - return rb_val; + return retval; } VALUE semian_simple_integer_value_get(VALUE self) { semian_simple_integer_t *res; - VALUE rb_val; + VALUE retval; TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); sem_meta_lock(res->sem_id); { semian_simple_integer_shared_t *data = get_value(res); - rb_val = RB_INT2NUM(data->val); + retval = RB_INT2NUM(data->val); } sem_meta_unlock(res->sem_id); - return rb_val; + return retval; } VALUE semian_simple_integer_value_set(VALUE self, VALUE val) { semian_simple_integer_t *res; - VALUE rb_val; + VALUE retval; TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); @@ -134,9 +134,9 @@ VALUE semian_simple_integer_value_set(VALUE self, VALUE val) { // TODO(michaelkipper): Check for respond_to?(:to_i) before calling. VALUE to_i = rb_funcall(val, rb_intern("to_i"), 0); data->val = RB_NUM2INT(to_i); - rb_val = RB_INT2NUM(data->val); + retval = RB_INT2NUM(data->val); } sem_meta_unlock(res->sem_id); - return rb_val; + return retval; } From 7ba1e9e964a3e3ac7349459c8a9d03d27bdaad93 Mon Sep 17 00:00:00 2001 From: Spike Lindsey Date: Fri, 14 Jun 2019 10:52:15 -0400 Subject: [PATCH 14/49] Change flag for choosing circuit implementation --- ext/semian/semian.c | 14 +++++++++++++- ext/semian/simple_integer.c | 2 +- test/simple_integer_test.rb | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/ext/semian/semian.c b/ext/semian/semian.c index 94910f94..b56f1120 100644 --- a/ext/semian/semian.c +++ b/ext/semian/semian.c @@ -1,5 +1,7 @@ #include "semian.h" +static int use_c_circuits(); + void Init_semian() { VALUE cSemian, cResource; @@ -65,9 +67,19 @@ void Init_semian() /* Maximum number of tickets available on this system. */ rb_define_const(cSemian, "MAX_TICKETS", INT2FIX(system_max_semaphore_count)); - if (getenv("USE_RUBY_CIRCUITS") == NULL || strcmp(getenv("USE_RUBY_CIRCUITS"), "false")) { + if (use_c_circuits()) { Init_SimpleInteger(); Init_SlidingWindow(); Init_CircuitBreaker(); } } + +static int +use_c_circuits() { + char *circuit_impl = getenv("CIRCUIT_IMPL"); + if (circuit_impl == NULL || strcmp(circuit_impl, "ruby") != 0) { + return 1; + } else { + return 0; + } +} diff --git a/ext/semian/simple_integer.c b/ext/semian/simple_integer.c index 94676ca2..7e9a35d3 100644 --- a/ext/semian/simple_integer.c +++ b/ext/semian/simple_integer.c @@ -25,7 +25,7 @@ static semian_simple_integer_shared_t* get_value(semian_simple_integer_t* res) { void Init_SimpleInteger() { - printf("[DEBUG] Init_SimpleInteger\n"); + dprintf("[DEBUG] Init_SimpleInteger\n"); VALUE cSemian = rb_const_get(rb_cObject, rb_intern("Semian")); VALUE cSimple = rb_const_get(cSemian, rb_intern("Simple")); diff --git a/test/simple_integer_test.rb b/test/simple_integer_test.rb index 4e8b76e4..29d72e28 100644 --- a/test/simple_integer_test.rb +++ b/test/simple_integer_test.rb @@ -42,7 +42,7 @@ def test_reset assert_equal(0, @integer.value) end - unless ENV['USE_RUBY_CIRCUITS'] + if ENV['CIRCUIT_IMPL'] != 'ruby' # Without locks, this only passes around 1 in every 5 runs def test_increment_race process_count = 255 From 8e64bd016fa173e8c2f5a7473ea731e814d31212 Mon Sep 17 00:00:00 2001 From: Spike Lindsey Date: Fri, 14 Jun 2019 11:37:09 -0400 Subject: [PATCH 15/49] Clarify flag scope --- ext/semian/semian.c | 2 +- ext/semian/simple_integer.c | 2 +- test/simple_integer_test.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/semian/semian.c b/ext/semian/semian.c index b56f1120..b21a7f64 100644 --- a/ext/semian/semian.c +++ b/ext/semian/semian.c @@ -76,7 +76,7 @@ void Init_semian() static int use_c_circuits() { - char *circuit_impl = getenv("CIRCUIT_IMPL"); + char *circuit_impl = getenv("SEMIAN_CIRCUIT_BREAKER_IMPL"); if (circuit_impl == NULL || strcmp(circuit_impl, "ruby") != 0) { return 1; } else { diff --git a/ext/semian/simple_integer.c b/ext/semian/simple_integer.c index 7e9a35d3..797b03be 100644 --- a/ext/semian/simple_integer.c +++ b/ext/semian/simple_integer.c @@ -25,7 +25,7 @@ static semian_simple_integer_shared_t* get_value(semian_simple_integer_t* res) { void Init_SimpleInteger() { - dprintf("[DEBUG] Init_SimpleInteger\n"); + dprintf("Init_SimpleInteger\n"); VALUE cSemian = rb_const_get(rb_cObject, rb_intern("Semian")); VALUE cSimple = rb_const_get(cSemian, rb_intern("Simple")); diff --git a/test/simple_integer_test.rb b/test/simple_integer_test.rb index 29d72e28..e0d6da57 100644 --- a/test/simple_integer_test.rb +++ b/test/simple_integer_test.rb @@ -42,7 +42,7 @@ def test_reset assert_equal(0, @integer.value) end - if ENV['CIRCUIT_IMPL'] != 'ruby' + if ENV['SEMIAN_CIRCUIT_BREAKER_IMPL'] != 'ruby' # Without locks, this only passes around 1 in every 5 runs def test_increment_race process_count = 255 From def383de617ae17fe97ff207736835adfea0b682 Mon Sep 17 00:00:00 2001 From: Spike Lindsey Date: Fri, 14 Jun 2019 15:16:56 -0400 Subject: [PATCH 16/49] Use semaphore locks around sliding window --- ext/semian/sliding_window.c | 138 +++++++++++++++++++++++------------- ext/semian/types.h | 1 + 2 files changed, 91 insertions(+), 48 deletions(-) diff --git a/ext/semian/sliding_window.c b/ext/semian/sliding_window.c index 1f370394..2009417e 100644 --- a/ext/semian/sliding_window.c +++ b/ext/semian/sliding_window.c @@ -4,6 +4,7 @@ #include #include #include "util.h" +#include "sysv_semaphores.h" static const rb_data_type_t semian_simple_sliding_window_type; @@ -85,62 +86,88 @@ VALUE semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_ dprintf(" key:%lu addr:0x%p max_size:%d length:%d start:%d end:%d", res->key, window, window->max_size, window->length, window->start, window->end); + res->sem_id = initialize_single_semaphore(res->key, SEM_DEFAULT_PERMISSIONS); return self; } VALUE semian_simple_sliding_window_size(VALUE self) { + VALUE retval; dprintf("semian_simple_sliding_window_size"); semian_simple_sliding_window_t *res; TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); - semian_simple_sliding_window_shared_t *window = get_window(res->key); - dprintf(" key:%lu addr:0x%p max_size:%d length:%d start:%d end:%d", res->key, window, window->max_size, window->length, window->start, window->end); + sem_meta_lock(res->sem_id); + { + semian_simple_sliding_window_shared_t *window = get_window(res->key); + dprintf(" key:%lu addr:0x%p max_size:%d length:%d start:%d end:%d", res->key, window, window->max_size, window->length, window->start, window->end); + retval = RB_INT2NUM(window->length); + } + sem_meta_unlock(res->sem_id); - return RB_INT2NUM(window->length); + return retval; } VALUE semian_simple_sliding_window_max_size(VALUE self) { + VALUE retval; dprintf("semian_simple_sliding_window_max_size"); semian_simple_sliding_window_t *res; TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); - semian_simple_sliding_window_shared_t *window = get_window(res->key); - dprintf(" key:%lu addr:0x%p max_size:%d length:%d start:%d end:%d", res->key, window, window->max_size, window->length, window->start, window->end); + sem_meta_lock(res->sem_id); + { + semian_simple_sliding_window_shared_t *window = get_window(res->key); + dprintf(" key:%lu addr:0x%p max_size:%d length:%d start:%d end:%d", res->key, window, window->max_size, window->length, window->start, window->end); + retval = RB_INT2NUM(window->max_size); + } + sem_meta_unlock(res->sem_id); - return RB_INT2NUM(window->max_size); + return retval; } VALUE semian_simple_sliding_window_values(VALUE self) { + VALUE retval; dprintf("semian_simple_sliding_window_values"); semian_simple_sliding_window_t *res; TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); - semian_simple_sliding_window_shared_t *window = get_window(res->key); - VALUE retval = rb_ary_new_capa(window->length); - for (int i = 0; i < window->length; ++i) { - int index = (window->start + i) % window->max_size; - int value = window->data[index]; - dprintf(" i:%d index: %d value:%d max_size:%d length:%d start:%d end:%d", i, index, value, window->max_size, window->length, window->start, window->end); - rb_ary_store(retval, i, RB_INT2NUM(value)); + sem_meta_lock(res->sem_id); + { + semian_simple_sliding_window_shared_t *window = get_window(res->key); + + retval = rb_ary_new_capa(window->length); + for (int i = 0; i < window->length; ++i) { + int index = (window->start + i) % window->max_size; + int value = window->data[index]; + dprintf(" i:%d index: %d value:%d max_size:%d length:%d start:%d end:%d", i, index, value, window->max_size, window->length, window->start, window->end); + rb_ary_store(retval, i, RB_INT2NUM(value)); + } } + sem_meta_unlock(res->sem_id); return retval; } VALUE semian_simple_sliding_window_last(VALUE self) { + VALUE retval; dprintf("semian_simple_sliding_window_last"); semian_simple_sliding_window_t *res; TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); - semian_simple_sliding_window_shared_t *window = get_window(res->key); - int index = (window->start + window->length - 1) % window->max_size; - dprintf(" index:%d last:%d", index, window->data[index]); + sem_meta_lock(res->sem_id); + { + semian_simple_sliding_window_shared_t *window = get_window(res->key); + + int index = (window->start + window->length - 1) % window->max_size; + dprintf(" index:%d last:%d", index, window->data[index]); + retval = RB_INT2NUM(window->data[index]); + } + sem_meta_unlock(res->sem_id); - return RB_INT2NUM(window->data[index]); + return retval; } VALUE semian_simple_sliding_window_clear(VALUE self) { @@ -148,11 +175,16 @@ VALUE semian_simple_sliding_window_clear(VALUE self) { semian_simple_sliding_window_t *res; TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); - semian_simple_sliding_window_shared_t *window = get_window(res->key); - window->length = 0; - window->start = 0; - window->end = 0; + sem_meta_lock(res->sem_id); + { + semian_simple_sliding_window_shared_t *window = get_window(res->key); + + window->length = 0; + window->start = 0; + window->end = 0; + } + sem_meta_unlock(res->sem_id); return self; } @@ -164,27 +196,33 @@ VALUE semian_simple_sliding_window_reject(VALUE self) { semian_simple_sliding_window_t *res; TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); - semian_simple_sliding_window_shared_t *window = get_window(res->key); - // Store these values because we're going to be modifying the buffer. - int start = window->start; - int length = window->length; - - int cleared = 0; - for (int i = 0; i < length; ++i) { - int index = (start + i) % length; - int value = window->data[index]; - dprintf(" i:%d index: %d value:%d max_size:%d length:%d start:%d end:%d", i, index, value, window->max_size, window->length, window->start, window->end); - VALUE y = rb_yield(RB_INT2NUM(value)); - if (RTEST(y)) { - if (cleared++ != i) { - rb_raise(rb_eArgError, "reject! must delete monotonically"); + sem_meta_lock(res->sem_id); + { + semian_simple_sliding_window_shared_t *window = get_window(res->key); + + // Store these values because we're going to be modifying the buffer. + int start = window->start; + int length = window->length; + + int cleared = 0; + for (int i = 0; i < length; ++i) { + int index = (start + i) % length; + int value = window->data[index]; + dprintf(" i:%d index: %d value:%d max_size:%d length:%d start:%d end:%d", i, index, value, window->max_size, window->length, window->start, window->end); + VALUE y = rb_yield(RB_INT2NUM(value)); + if (RTEST(y)) { + if (cleared++ != i) { + sem_meta_unlock(res->sem_id); + rb_raise(rb_eArgError, "reject! must delete monotonically"); + } + window->start = (window->start + 1) % window->length; + window->length--; + dprintf(" Removed index:%d (val:%d)", i, value); } - window->start = (window->start + 1) % window->length; - window->length--; - dprintf(" Removed index:%d (val:%d)", i, value); } } + sem_meta_unlock(res->sem_id); return self; } @@ -195,18 +233,22 @@ VALUE semian_simple_sliding_window_push(VALUE self, VALUE value) { semian_simple_sliding_window_t *res; TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); - semian_simple_sliding_window_shared_t *window = get_window(res->key); - if (window->length == window->max_size) { - window->length--; - window->start = (window->start + 1) % window->max_size; - } + sem_meta_lock(res->sem_id); + { + semian_simple_sliding_window_shared_t *window = get_window(res->key); + if (window->length == window->max_size) { + window->length--; + window->start = (window->start + 1) % window->max_size; + } - const int index = window->end; - window->length++; - window->data[index] = RB_NUM2INT(value); - window->end = (window->end + 1) % window->max_size; + const int index = window->end; + window->length++; + window->data[index] = RB_NUM2INT(value); + window->end = (window->end + 1) % window->max_size; - dprintf(" Pushed val:%d index:%d max_size:%d length:%d start:%d end:%d", RB_NUM2INT(value), index, window->max_size, window->length, window->start, window->end); + dprintf(" Pushed val:%d index:%d max_size:%d length:%d start:%d end:%d", RB_NUM2INT(value), index, window->max_size, window->length, window->start, window->end); + } + sem_meta_unlock(res->sem_id); return self; } diff --git a/ext/semian/types.h b/ext/semian/types.h index 68b94e6b..3dade21d 100644 --- a/ext/semian/types.h +++ b/ext/semian/types.h @@ -67,6 +67,7 @@ typedef struct { // Internal simple sliding window structure typedef struct { uint64_t key; + int sem_id; } semian_simple_sliding_window_t; // Shared simple sliding window structure From efb62a9e681f00c90d67d2636c6e93b80d6d8f0d Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Fri, 14 Jun 2019 15:42:28 -0400 Subject: [PATCH 17/49] Stylistic changes --- ext/semian/circuit_breaker.c | 10 ++-- ext/semian/simple_integer.c | 29 +++++++--- ext/semian/sliding_window.c | 102 ++++++++++++++++++----------------- 3 files changed, 79 insertions(+), 62 deletions(-) diff --git a/ext/semian/circuit_breaker.c b/ext/semian/circuit_breaker.c index 6bd3761b..d4e24567 100644 --- a/ext/semian/circuit_breaker.c +++ b/ext/semian/circuit_breaker.c @@ -8,7 +8,8 @@ static const rb_data_type_t semian_circuit_breaker_type; -void Init_CircuitBreaker() { +void +Init_CircuitBreaker() { dprintf("Init_CircuitBreaker"); VALUE cSemian = rb_const_get(rb_cObject, rb_intern("Semian")); @@ -18,7 +19,8 @@ void Init_CircuitBreaker() { rb_define_method(cCircuitBreaker, "initialize_circuit_breaker", semian_circuit_breaker_initialize, 1); } -VALUE semian_circuit_breaker_alloc(VALUE klass) +VALUE +semian_circuit_breaker_alloc(VALUE klass) { dprintf("semian_circuit_breaker_alloc"); @@ -27,10 +29,10 @@ VALUE semian_circuit_breaker_alloc(VALUE klass) return obj; } -VALUE semian_circuit_breaker_initialize(VALUE self, VALUE id) +VALUE +semian_circuit_breaker_initialize(VALUE self, VALUE id) { const char *c_id_str = check_id_arg(id); - dprintf("semian_circuit_breaker_initialize('%s')", c_id_str); // Build semian resource structure diff --git a/ext/semian/simple_integer.c b/ext/semian/simple_integer.c index 797b03be..e698931e 100644 --- a/ext/semian/simple_integer.c +++ b/ext/semian/simple_integer.c @@ -9,7 +9,9 @@ static const rb_data_type_t semian_simple_integer_type; -static semian_simple_integer_shared_t* get_value(semian_simple_integer_t* res) { +static semian_simple_integer_shared_t* +get_value(semian_simple_integer_t* res) +{ int shmid = shmget(res->key, sizeof(semian_simple_integer_shared_t), IPC_CREAT | SEM_DEFAULT_PERMISSIONS); if (shmid == -1) { rb_raise(rb_eArgError, "could not create shared memory (%s)", strerror(errno)); @@ -23,7 +25,8 @@ static semian_simple_integer_shared_t* get_value(semian_simple_integer_t* res) { return (semian_simple_integer_shared_t*)val; } -void Init_SimpleInteger() +void +Init_SimpleInteger() { dprintf("Init_SimpleInteger\n"); @@ -39,7 +42,8 @@ void Init_SimpleInteger() rb_define_method(cSimpleInteger, "value=", semian_simple_integer_value_set, 1); } -VALUE semian_simple_integer_alloc(VALUE klass) +VALUE +semian_simple_integer_alloc(VALUE klass) { semian_simple_integer_t *res; VALUE obj = TypedData_Make_Struct(klass, semian_simple_integer_t, &semian_simple_integer_type, res); @@ -47,7 +51,8 @@ VALUE semian_simple_integer_alloc(VALUE klass) } -VALUE semian_simple_integer_initialize(VALUE self, VALUE name) +VALUE +semian_simple_integer_initialize(VALUE self, VALUE name) { semian_simple_integer_t *res; TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); @@ -61,7 +66,9 @@ VALUE semian_simple_integer_initialize(VALUE self, VALUE name) return self; } -VALUE semian_simple_integer_increment(int argc, VALUE *argv, VALUE self) { +VALUE +semian_simple_integer_increment(int argc, VALUE *argv, VALUE self) +{ // This is definitely the worst API ever. // https://silverhammermba.github.io/emberb/c/#parsing-arguments VALUE val; @@ -88,7 +95,9 @@ VALUE semian_simple_integer_increment(int argc, VALUE *argv, VALUE self) { return retval; } -VALUE semian_simple_integer_reset(VALUE self) { +VALUE +semian_simple_integer_reset(VALUE self) +{ semian_simple_integer_t *res; VALUE retval; @@ -105,7 +114,9 @@ VALUE semian_simple_integer_reset(VALUE self) { return retval; } -VALUE semian_simple_integer_value_get(VALUE self) { +VALUE +semian_simple_integer_value_get(VALUE self) +{ semian_simple_integer_t *res; VALUE retval; @@ -121,7 +132,9 @@ VALUE semian_simple_integer_value_get(VALUE self) { return retval; } -VALUE semian_simple_integer_value_set(VALUE self, VALUE val) { +VALUE +semian_simple_integer_value_set(VALUE self, VALUE val) +{ semian_simple_integer_t *res; VALUE retval; diff --git a/ext/semian/sliding_window.c b/ext/semian/sliding_window.c index 2009417e..2b97e664 100644 --- a/ext/semian/sliding_window.c +++ b/ext/semian/sliding_window.c @@ -8,7 +8,9 @@ static const rb_data_type_t semian_simple_sliding_window_type; -static semian_simple_sliding_window_shared_t* get_window(uint64_t key) { +static semian_simple_sliding_window_shared_t* +get_window(uint64_t key) +{ const int permissions = 0664; int shmid = shmget(key, sizeof(int), IPC_CREAT | permissions); if (shmid == -1) { @@ -23,7 +25,8 @@ static semian_simple_sliding_window_shared_t* get_window(uint64_t key) { return (semian_simple_sliding_window_shared_t*)val; } -static int check_max_size_arg(VALUE max_size) +static int +check_max_size_arg(VALUE max_size) { int retval = -1; switch (TYPE(max_size)) { @@ -45,7 +48,18 @@ static int check_max_size_arg(VALUE max_size) return retval; } -void Init_SlidingWindow() { +// Get the C object for a Ruby instance +static semian_simple_sliding_window_t* +get_object(VALUE self) +{ + semian_simple_sliding_window_t *res; + TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); + return res; +} + +void +Init_SlidingWindow() +{ dprintf("Init_SlidingWindow"); VALUE cSemian = rb_const_get(rb_cObject, rb_intern("Semian")); @@ -63,17 +77,18 @@ void Init_SlidingWindow() { rb_define_method(cSlidingWindow, "reject!", semian_simple_sliding_window_reject, 0); } -VALUE semian_simple_sliding_window_alloc(VALUE klass) { +VALUE +semian_simple_sliding_window_alloc(VALUE klass) +{ semian_simple_sliding_window_t *res; VALUE obj = TypedData_Make_Struct(klass, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); return obj; } -VALUE semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_size) { - dprintf("semian_simple_sliding_window_initialize"); - - semian_simple_sliding_window_t *res; - TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); +VALUE +semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_size) +{ + semian_simple_sliding_window_t *res = get_object(self); const char *id_str = check_id_arg(name); res->key = generate_key(id_str); @@ -84,18 +99,15 @@ VALUE semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_ window->start = 0; window->end = 0; - dprintf(" key:%lu addr:0x%p max_size:%d length:%d start:%d end:%d", res->key, window, window->max_size, window->length, window->start, window->end); - res->sem_id = initialize_single_semaphore(res->key, SEM_DEFAULT_PERMISSIONS); return self; } -VALUE semian_simple_sliding_window_size(VALUE self) { +VALUE +semian_simple_sliding_window_size(VALUE self) +{ + semian_simple_sliding_window_t *res = get_object(self); VALUE retval; - dprintf("semian_simple_sliding_window_size"); - - semian_simple_sliding_window_t *res; - TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); sem_meta_lock(res->sem_id); { @@ -108,12 +120,11 @@ VALUE semian_simple_sliding_window_size(VALUE self) { return retval; } -VALUE semian_simple_sliding_window_max_size(VALUE self) { +VALUE +semian_simple_sliding_window_max_size(VALUE self) +{ + semian_simple_sliding_window_t *res = get_object(self); VALUE retval; - dprintf("semian_simple_sliding_window_max_size"); - - semian_simple_sliding_window_t *res; - TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); sem_meta_lock(res->sem_id); { @@ -126,12 +137,11 @@ VALUE semian_simple_sliding_window_max_size(VALUE self) { return retval; } -VALUE semian_simple_sliding_window_values(VALUE self) { +VALUE +semian_simple_sliding_window_values(VALUE self) +{ + semian_simple_sliding_window_t *res = get_object(self); VALUE retval; - dprintf("semian_simple_sliding_window_values"); - - semian_simple_sliding_window_t *res; - TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); sem_meta_lock(res->sem_id); { @@ -150,19 +160,17 @@ VALUE semian_simple_sliding_window_values(VALUE self) { return retval; } -VALUE semian_simple_sliding_window_last(VALUE self) { +VALUE +semian_simple_sliding_window_last(VALUE self) +{ + semian_simple_sliding_window_t *res = get_object(self); VALUE retval; - dprintf("semian_simple_sliding_window_last"); - - semian_simple_sliding_window_t *res; - TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); sem_meta_lock(res->sem_id); { semian_simple_sliding_window_shared_t *window = get_window(res->key); int index = (window->start + window->length - 1) % window->max_size; - dprintf(" index:%d last:%d", index, window->data[index]); retval = RB_INT2NUM(window->data[index]); } sem_meta_unlock(res->sem_id); @@ -170,11 +178,10 @@ VALUE semian_simple_sliding_window_last(VALUE self) { return retval; } -VALUE semian_simple_sliding_window_clear(VALUE self) { - dprintf("semian_simple_sliding_window_clear"); - - semian_simple_sliding_window_t *res; - TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); +VALUE +semian_simple_sliding_window_clear(VALUE self) +{ + semian_simple_sliding_window_t *res = get_object(self); sem_meta_lock(res->sem_id); { @@ -189,14 +196,13 @@ VALUE semian_simple_sliding_window_clear(VALUE self) { return self; } -VALUE semian_simple_sliding_window_reject(VALUE self) { - dprintf("semian_simple_sliding_window_reject"); +VALUE +semian_simple_sliding_window_reject(VALUE self) +{ + semian_simple_sliding_window_t *res = get_object(self); rb_need_block(); - semian_simple_sliding_window_t *res; - TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); - sem_meta_lock(res->sem_id); { semian_simple_sliding_window_shared_t *window = get_window(res->key); @@ -209,7 +215,6 @@ VALUE semian_simple_sliding_window_reject(VALUE self) { for (int i = 0; i < length; ++i) { int index = (start + i) % length; int value = window->data[index]; - dprintf(" i:%d index: %d value:%d max_size:%d length:%d start:%d end:%d", i, index, value, window->max_size, window->length, window->start, window->end); VALUE y = rb_yield(RB_INT2NUM(value)); if (RTEST(y)) { if (cleared++ != i) { @@ -227,11 +232,10 @@ VALUE semian_simple_sliding_window_reject(VALUE self) { return self; } -VALUE semian_simple_sliding_window_push(VALUE self, VALUE value) { - dprintf("semian_simple_sliding_window_push"); - - semian_simple_sliding_window_t *res; - TypedData_Get_Struct(self, semian_simple_sliding_window_t, &semian_simple_sliding_window_type, res); +VALUE +semian_simple_sliding_window_push(VALUE self, VALUE value) +{ + semian_simple_sliding_window_t *res = get_object(self); sem_meta_lock(res->sem_id); { @@ -245,8 +249,6 @@ VALUE semian_simple_sliding_window_push(VALUE self, VALUE value) { window->length++; window->data[index] = RB_NUM2INT(value); window->end = (window->end + 1) % window->max_size; - - dprintf(" Pushed val:%d index:%d max_size:%d length:%d start:%d end:%d", RB_NUM2INT(value), index, window->max_size, window->length, window->start, window->end); } sem_meta_unlock(res->sem_id); From 7474bed872186c805c2ab6fbcb588946d27f4155 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Mon, 17 Jun 2019 17:42:18 -0400 Subject: [PATCH 18/49] Quick script to reset IPC state --- scripts/cleanup_ipc.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100755 scripts/cleanup_ipc.sh diff --git a/scripts/cleanup_ipc.sh b/scripts/cleanup_ipc.sh new file mode 100755 index 00000000..b491f249 --- /dev/null +++ b/scripts/cleanup_ipc.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +ME=`whoami` + +IPCS_S=`ipcs -s | egrep "0x[0-9a-f]+ [0-9]+" | grep $ME | cut -f2 -d" "` +IPCS_M=`ipcs -m | egrep "0x[0-9a-f]+ [0-9]+" | grep $ME | cut -f2 -d" "` +IPCS_Q=`ipcs -q | egrep "0x[0-9a-f]+ [0-9]+" | grep $ME | cut -f2 -d" "` + +for id in $IPCS_M; do + ipcrm -m $id; +done + +for id in $IPCS_S; do + ipcrm -s $id; +done + +for id in $IPCS_Q; do + ipcrm -q $id; +done + From 203cdfd2369738a9e4548968236faf68a86047d7 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Mon, 17 Jun 2019 17:44:10 -0400 Subject: [PATCH 19/49] More locks around shared data structures --- ext/semian/circuit_breaker.c | 66 +++++++------- ext/semian/simple_integer.c | 72 +++++++-------- ext/semian/sliding_window.c | 135 +++++++++++++++-------------- ext/semian/sysv_semaphores.c | 8 +- ext/semian/sysv_semaphores.h | 5 +- ext/semian/sysv_shared_memory.c | 54 ++++++++++++ ext/semian/sysv_shared_memory.h | 21 +++++ ext/semian/types.h | 25 +++--- ext/semian/util.h | 2 +- lib/semian/circuit_breaker.rb | 2 +- lib/semian/circuit_breaker.rb.orig | 18 ++-- 11 files changed, 246 insertions(+), 162 deletions(-) create mode 100644 ext/semian/sysv_shared_memory.c create mode 100644 ext/semian/sysv_shared_memory.h diff --git a/ext/semian/circuit_breaker.c b/ext/semian/circuit_breaker.c index d4e24567..f689cabe 100644 --- a/ext/semian/circuit_breaker.c +++ b/ext/semian/circuit_breaker.c @@ -1,12 +1,33 @@ #include "circuit_breaker.h" -#include -#include -#include +#include "sysv_semaphores.h" +#include "sysv_shared_memory.h" #include "types.h" #include "util.h" -static const rb_data_type_t semian_circuit_breaker_type; +void +semian_circuit_breaker_free(void* ptr) +{ + semian_circuit_breaker_t* res = (semian_circuit_breaker_t*)ptr; + free_shared_memory(res->shmem); +} + +size_t +semian_circuit_breaker_size(const void* ptr) +{ + return sizeof(semian_circuit_breaker_t); +} + +static const rb_data_type_t semian_circuit_breaker_type = { + .wrap_struct_name = "semian_circuit_breaker", + .function = { + .dmark = NULL, + .dfree = semian_circuit_breaker_free, + .dsize = semian_circuit_breaker_size, + }, + .data = NULL, + .flags = RUBY_TYPED_FREE_IMMEDIATELY, +}; void Init_CircuitBreaker() { @@ -22,48 +43,21 @@ Init_CircuitBreaker() { VALUE semian_circuit_breaker_alloc(VALUE klass) { - dprintf("semian_circuit_breaker_alloc"); - semian_circuit_breaker_t *res; VALUE obj = TypedData_Make_Struct(klass, semian_circuit_breaker_t, &semian_circuit_breaker_type, res); return obj; } VALUE -semian_circuit_breaker_initialize(VALUE self, VALUE id) +semian_circuit_breaker_initialize(VALUE self, VALUE name) { - const char *c_id_str = check_id_arg(id); - dprintf("semian_circuit_breaker_initialize('%s')", c_id_str); - - // Build semian resource structure semian_circuit_breaker_t *res = NULL; TypedData_Get_Struct(self, semian_circuit_breaker_t, &semian_circuit_breaker_type, res); + res->key = generate_key(to_s(name)); - // Initialize the semaphore set - // initialize_semaphore_set(res, c_id_str, c_permissions, c_tickets, c_quota); - res->name = strdup(c_id_str); - - key_t key = generate_key(res->name); - dprintf("Creating shared memory for '%s' (id %u)", res->name, key); - const int permissions = 0664; - int shmid = shmget(key, 1024, IPC_CREAT | permissions); - if (shmid == -1) { - rb_raise(rb_eArgError, "could not create shared memory (%s)", strerror(errno)); - } - - dprintf("Getting shared memory (id %u)", shmid); - void *val = shmat(shmid, NULL, 0); - if (val == (void*)-1) { - rb_raise(rb_eArgError, "could not get shared memory (%s)", strerror(errno)); - } - - semian_circuit_breaker_shared_t *data = (semian_circuit_breaker_shared_t*)val; - if (data == NULL) { - rb_raise(rb_eArgError, "could not get shared memory (%s)", strerror(errno)); - } - - dprintf("successes = %d", data->successes); - data->successes = 0; + dprintf("Initializing circuit breaker '%s' (key: %lu)", to_s(name), res->key); + res->sem_id = initialize_single_semaphore(res->key, SEM_DEFAULT_PERMISSIONS); + res->shmem = get_or_create_shared_memory(res->key, NULL); return self; } diff --git a/ext/semian/simple_integer.c b/ext/semian/simple_integer.c index e698931e..7ba28398 100644 --- a/ext/semian/simple_integer.c +++ b/ext/semian/simple_integer.c @@ -1,28 +1,38 @@ #include "simple_integer.h" -#include -#include -#include +#include "sysv_semaphores.h" +#include "sysv_shared_memory.h" #include "types.h" #include "util.h" -#include "sysv_semaphores.h" -static const rb_data_type_t semian_simple_integer_type; - -static semian_simple_integer_shared_t* -get_value(semian_simple_integer_t* res) +void +semian_simple_integer_dfree(void* ptr) { - int shmid = shmget(res->key, sizeof(semian_simple_integer_shared_t), IPC_CREAT | SEM_DEFAULT_PERMISSIONS); - if (shmid == -1) { - rb_raise(rb_eArgError, "could not create shared memory (%s)", strerror(errno)); - } + semian_simple_integer_t* res = (semian_simple_integer_t*)ptr; + free_shared_memory(res->shmem); +} - void *val = shmat(shmid, NULL, 0); - if (val == (void*)-1) { - rb_raise(rb_eArgError, "could not get shared memory (%s)", strerror(errno)); - } +size_t +semian_simple_integer_dsize(const void* ptr) +{ + return sizeof(semian_simple_integer_t); +} - return (semian_simple_integer_shared_t*)val; +static const rb_data_type_t semian_simple_integer_type = { + .wrap_struct_name = "semian_simple_integer", + .function = { + .dmark = NULL, + .dfree = semian_simple_integer_dfree, + .dsize = semian_simple_integer_dsize, + }, + .data = NULL, + .flags = RUBY_TYPED_FREE_IMMEDIATELY, +}; + +static void init_fn(void* ptr) +{ + semian_simple_integer_shared_t* res = (semian_simple_integer_shared_t*)ptr; + res->val = 0; } void @@ -50,7 +60,6 @@ semian_simple_integer_alloc(VALUE klass) return obj; } - VALUE semian_simple_integer_initialize(VALUE self, VALUE name) { @@ -58,10 +67,9 @@ semian_simple_integer_initialize(VALUE self, VALUE name) TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); res->key = generate_key(to_s(name)); - semian_simple_integer_shared_t* data = get_value(res); - data->val = 0; - + dprintf("Initializing simple integer '%s' (key: %lu)", to_s(name), res->key); res->sem_id = initialize_single_semaphore(res->key, SEM_DEFAULT_PERMISSIONS); + res->shmem = get_or_create_shared_memory(res->key, &init_fn); return self; } @@ -81,14 +89,12 @@ semian_simple_integer_increment(int argc, VALUE *argv, VALUE self) sem_meta_lock(res->sem_id); { - semian_simple_integer_shared_t *data = get_value(res); - if (NIL_P(val)) { - data->val += 1; + res->shmem->val += 1; } else { - data->val += RB_NUM2INT(val); + res->shmem->val += RB_NUM2INT(val); } - retval = RB_INT2NUM(data->val); + retval = RB_INT2NUM(res->shmem->val); } sem_meta_unlock(res->sem_id); @@ -105,9 +111,8 @@ semian_simple_integer_reset(VALUE self) sem_meta_lock(res->sem_id); { - semian_simple_integer_shared_t *data = get_value(res); - data->val = 0; - retval = RB_INT2NUM(data->val); + res->shmem->val = 0; + retval = RB_INT2NUM(res->shmem->val); } sem_meta_unlock(res->sem_id); @@ -124,8 +129,7 @@ semian_simple_integer_value_get(VALUE self) sem_meta_lock(res->sem_id); { - semian_simple_integer_shared_t *data = get_value(res); - retval = RB_INT2NUM(data->val); + retval = RB_INT2NUM(res->shmem->val); } sem_meta_unlock(res->sem_id); @@ -142,12 +146,10 @@ semian_simple_integer_value_set(VALUE self, VALUE val) sem_meta_lock(res->sem_id); { - semian_simple_integer_shared_t *data = get_value(res); - // TODO(michaelkipper): Check for respond_to?(:to_i) before calling. VALUE to_i = rb_funcall(val, rb_intern("to_i"), 0); - data->val = RB_NUM2INT(to_i); - retval = RB_INT2NUM(data->val); + res->shmem->val = RB_NUM2INT(to_i); + retval = RB_INT2NUM(res->shmem->val); } sem_meta_unlock(res->sem_id); diff --git a/ext/semian/sliding_window.c b/ext/semian/sliding_window.c index 2b97e664..89dbf253 100644 --- a/ext/semian/sliding_window.c +++ b/ext/semian/sliding_window.c @@ -1,28 +1,40 @@ #include "sliding_window.h" -#include -#include -#include #include "util.h" #include "sysv_semaphores.h" +#include "sysv_shared_memory.h" -static const rb_data_type_t semian_simple_sliding_window_type; - -static semian_simple_sliding_window_shared_t* -get_window(uint64_t key) +void +semian_simple_sliding_window_dfree(void* ptr) { - const int permissions = 0664; - int shmid = shmget(key, sizeof(int), IPC_CREAT | permissions); - if (shmid == -1) { - rb_raise(rb_eArgError, "could not create shared memory (%s)", strerror(errno)); - } + semian_simple_sliding_window_t* res = (semian_simple_sliding_window_t*)ptr; + free_shared_memory(res->shmem); +} - void *val = shmat(shmid, NULL, 0); - if (val == (void*)-1) { - rb_raise(rb_eArgError, "could not get shared memory (%s)", strerror(errno)); - } +size_t +semian_simple_sliding_window_dsize(const void* ptr) +{ + return sizeof(semian_simple_sliding_window_t); +} - return (semian_simple_sliding_window_shared_t*)val; +static const rb_data_type_t semian_simple_sliding_window_type = { + .wrap_struct_name = "semian_simple_sliding_window", + .function = { + .dmark = NULL, + .dfree = semian_simple_sliding_window_dfree, + .dsize = semian_simple_sliding_window_dsize, + }, + .data = NULL, + .flags = RUBY_TYPED_FREE_IMMEDIATELY, +}; + +static void init_fn(void* ptr) +{ + semian_simple_sliding_window_shared_t* res = (semian_simple_sliding_window_shared_t*)ptr; + res->max_size = 0; + res->length = 0; + res->start = 0; + res->end = 0; } static int @@ -89,17 +101,29 @@ VALUE semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_size) { semian_simple_sliding_window_t *res = get_object(self); + res->key = generate_key(to_s(name)); - const char *id_str = check_id_arg(name); - res->key = generate_key(id_str); + dprintf("Initializing simple sliding window '%s' (key: %lu)", to_s(name), res->key); + res->sem_id = initialize_single_semaphore(res->key, SEM_DEFAULT_PERMISSIONS); + res->shmem = get_or_create_shared_memory(res->key, init_fn); - semian_simple_sliding_window_shared_t *window = get_window(res->key); - window->max_size = check_max_size_arg(max_size); - window->length = 0; - window->start = 0; - window->end = 0; + int max_size_val = check_max_size_arg(max_size); + + sem_meta_lock(res->sem_id); + { + if (res->shmem->max_size == 0) { + dprintf("Setting max_size for '%s' to %d", to_s(name), max_size_val); + res->shmem->max_size = max_size_val; + } else if (res->shmem->max_size != max_size_val) { + // TODO(michaelkipper): Figure out what do do in this case... + printf("Warning: Max size of %d is different than current value of %d", max_size_val, res->shmem->max_size); + // dprintf("max_size %d is different than %d", max_size_val, res->shmem->max_size); + // sem_meta_unlock(res->sem_id); + // rb_raise(rb_eArgError, "max_size was different"); + } + } + sem_meta_unlock(res->sem_id); - res->sem_id = initialize_single_semaphore(res->key, SEM_DEFAULT_PERMISSIONS); return self; } @@ -111,9 +135,7 @@ semian_simple_sliding_window_size(VALUE self) sem_meta_lock(res->sem_id); { - semian_simple_sliding_window_shared_t *window = get_window(res->key); - dprintf(" key:%lu addr:0x%p max_size:%d length:%d start:%d end:%d", res->key, window, window->max_size, window->length, window->start, window->end); - retval = RB_INT2NUM(window->length); + retval = RB_INT2NUM(res->shmem->length); } sem_meta_unlock(res->sem_id); @@ -128,9 +150,7 @@ semian_simple_sliding_window_max_size(VALUE self) sem_meta_lock(res->sem_id); { - semian_simple_sliding_window_shared_t *window = get_window(res->key); - dprintf(" key:%lu addr:0x%p max_size:%d length:%d start:%d end:%d", res->key, window, window->max_size, window->length, window->start, window->end); - retval = RB_INT2NUM(window->max_size); + retval = RB_INT2NUM(res->shmem->max_size); } sem_meta_unlock(res->sem_id); @@ -145,13 +165,10 @@ semian_simple_sliding_window_values(VALUE self) sem_meta_lock(res->sem_id); { - semian_simple_sliding_window_shared_t *window = get_window(res->key); - - retval = rb_ary_new_capa(window->length); - for (int i = 0; i < window->length; ++i) { - int index = (window->start + i) % window->max_size; - int value = window->data[index]; - dprintf(" i:%d index: %d value:%d max_size:%d length:%d start:%d end:%d", i, index, value, window->max_size, window->length, window->start, window->end); + retval = rb_ary_new_capa(res->shmem->length); + for (int i = 0; i < res->shmem->length; ++i) { + int index = (res->shmem->start + i) % res->shmem->max_size; + int value = res->shmem->data[index]; rb_ary_store(retval, i, RB_INT2NUM(value)); } } @@ -168,10 +185,8 @@ semian_simple_sliding_window_last(VALUE self) sem_meta_lock(res->sem_id); { - semian_simple_sliding_window_shared_t *window = get_window(res->key); - - int index = (window->start + window->length - 1) % window->max_size; - retval = RB_INT2NUM(window->data[index]); + int index = (res->shmem->start + res->shmem->length - 1) % res->shmem->max_size; + retval = RB_INT2NUM(res->shmem->data[index]); } sem_meta_unlock(res->sem_id); @@ -185,11 +200,9 @@ semian_simple_sliding_window_clear(VALUE self) sem_meta_lock(res->sem_id); { - semian_simple_sliding_window_shared_t *window = get_window(res->key); - - window->length = 0; - window->start = 0; - window->end = 0; + res->shmem->length = 0; + res->shmem->start = 0; + res->shmem->end = 0; } sem_meta_unlock(res->sem_id); @@ -205,25 +218,22 @@ semian_simple_sliding_window_reject(VALUE self) sem_meta_lock(res->sem_id); { - semian_simple_sliding_window_shared_t *window = get_window(res->key); - // Store these values because we're going to be modifying the buffer. - int start = window->start; - int length = window->length; + int start = res->shmem->start; + int length = res->shmem->length; int cleared = 0; for (int i = 0; i < length; ++i) { int index = (start + i) % length; - int value = window->data[index]; + int value = res->shmem->data[index]; VALUE y = rb_yield(RB_INT2NUM(value)); if (RTEST(y)) { if (cleared++ != i) { sem_meta_unlock(res->sem_id); rb_raise(rb_eArgError, "reject! must delete monotonically"); } - window->start = (window->start + 1) % window->length; - window->length--; - dprintf(" Removed index:%d (val:%d)", i, value); + res->shmem->start = (res->shmem->start + 1) % res->shmem->length; + res->shmem->length--; } } } @@ -239,16 +249,15 @@ semian_simple_sliding_window_push(VALUE self, VALUE value) sem_meta_lock(res->sem_id); { - semian_simple_sliding_window_shared_t *window = get_window(res->key); - if (window->length == window->max_size) { - window->length--; - window->start = (window->start + 1) % window->max_size; + if (res->shmem->length == res->shmem->max_size) { + res->shmem->length--; + res->shmem->start = (res->shmem->start + 1) % res->shmem->max_size; } - const int index = window->end; - window->length++; - window->data[index] = RB_NUM2INT(value); - window->end = (window->end + 1) % window->max_size; + const int index = res->shmem->end; + res->shmem->length++; + res->shmem->data[index] = RB_NUM2INT(value); + res->shmem->end = (res->shmem->end + 1) % res->shmem->max_size; } sem_meta_unlock(res->sem_id); diff --git a/ext/semian/sysv_semaphores.c b/ext/semian/sysv_semaphores.c index 79c884ce..030ca75c 100644 --- a/ext/semian/sysv_semaphores.c +++ b/ext/semian/sysv_semaphores.c @@ -7,7 +7,7 @@ static void * acquire_semaphore(void *p); static int -wait_for_new_semaphore_set(key_t key, long permissions); +wait_for_new_semaphore_set(uint64_t key, long permissions); static void initialize_new_semaphore_values(int sem_id, long permissions); @@ -43,6 +43,7 @@ initialize_semaphore_set(semian_resource_t* res, const char* id_str, long permis if (res->sem_id != -1) { // Happy path - we are the first worker, initialize the semaphore set. initialize_new_semaphore_values(res->sem_id, permissions); + dprintf("Created semaphore set (key:%lu sem_id:%d)", res->key, res->sem_id); } else { // Something went wrong if (errno != EEXIST) { @@ -199,7 +200,7 @@ initialize_new_semaphore_values(int sem_id, long permissions) } static int -wait_for_new_semaphore_set(key_t key, long permissions) +wait_for_new_semaphore_set(uint64_t key, long permissions) { int i; int sem_id = -1; @@ -246,7 +247,7 @@ diff_timespec_ms(struct timespec *end, struct timespec *begin) } int -initialize_single_semaphore(key_t key, long permissions) +initialize_single_semaphore(uint64_t key, long permissions) { int sem_id = semget(key, 1, IPC_CREAT | IPC_EXCL | permissions); @@ -260,6 +261,7 @@ initialize_single_semaphore(key_t key, long permissions) if (semctl(sem_id, 0, SETVAL, 1) == -1) { raise_semian_syscall_error("semctl()", errno); } + dprintf("Created semaphore (key:%lu sem_id:%d)", key, sem_id); } else { // Something went wrong if (errno != EEXIST) { diff --git a/ext/semian/sysv_semaphores.h b/ext/semian/sysv_semaphores.h index d0890c8c..9d4e2203 100644 --- a/ext/semian/sysv_semaphores.h +++ b/ext/semian/sysv_semaphores.h @@ -17,6 +17,7 @@ and functions associated directly weth semops. #include "types.h" #include "tickets.h" +#include "util.h" // Defines for ruby threading primitives #if defined(HAVE_RB_THREAD_CALL_WITHOUT_GVL) && defined(HAVE_RUBY_THREAD_H) @@ -112,13 +113,13 @@ acquire_semaphore_without_gvl(void *p); // Initializes a semaphore set with a single semaphore, for general purpose // locking int -initialize_single_semaphore(key_t key, long permissions); +initialize_single_semaphore(uint64_t key, long permissions); #ifdef DEBUG static inline void print_sem_vals(int sem_id) { - printf("[DEBUG] sem_id: %d, lock: %d, tickets: %d configured: %d, registered workers %d\n", + dprintf("sem_id: %d, lock: %d, tickets: %d configured: %d, registered workers %d", sem_id, get_sem_val(sem_id, SI_SEM_LOCK), get_sem_val(sem_id, SI_SEM_TICKETS), diff --git a/ext/semian/sysv_shared_memory.c b/ext/semian/sysv_shared_memory.c new file mode 100644 index 00000000..be1e4b7d --- /dev/null +++ b/ext/semian/sysv_shared_memory.c @@ -0,0 +1,54 @@ +#include "sysv_shared_memory.h" + +#include "util.h" + +#define TIMEOUT_MS (5 * 1e6) +#define WAIT_MS (10) +#define RETRIES (TIMEOUT_MS / WAIT_MS) + +void* +wait_for_shared_memory(uint64_t key) +{ + for (int i = 0; i < RETRIES; ++i) { + int shmid = shmget(key, SHM_DEFAULT_SIZE, SHM_DEFAULT_PERMISSIONS); + if (shmid != -1) { + return shmat(shmid, NULL, 0); + } + usleep(WAIT_MS); + } + + rb_raise(rb_eArgError, "could not get shared memory"); +} + +void* +get_or_create_shared_memory(uint64_t key, shared_memory_init_fn fn) +{ + void* shmem = NULL; + if (!key) return NULL; + + dprintf("Creating shared memory (key: %lu)", key); + int shmid = shmget(key, SHM_DEFAULT_SIZE, IPC_CREAT | IPC_EXCL | SHM_DEFAULT_PERMISSIONS); + if (shmid != -1) { + dprintf("Created shared memory (key: %lu)", key); + + shmem = shmat(shmid, NULL, 0); + if (shmem == (void*)-1) { + rb_raise(rb_eArgError, "could not get shared memory (%s)", strerror(errno)); + } + + if (fn) fn(shmem); + + shmctl(key, IPC_RMID, NULL); + } else { + shmem = wait_for_shared_memory(key); + } + + return shmem; +} + +void +free_shared_memory(void* shmem) +{ + if (!shmem) return; + shmdt(shmem); +} diff --git a/ext/semian/sysv_shared_memory.h b/ext/semian/sysv_shared_memory.h new file mode 100644 index 00000000..ee84560c --- /dev/null +++ b/ext/semian/sysv_shared_memory.h @@ -0,0 +1,21 @@ +#ifndef EXT_SEMIAN_SYSV_SHARED_MEMORY_H +#define EXT_SEMIAN_SYSV_SHARED_MEMORY_H + +#include +#include +#include +#include + +// Default permissions for shared memory +#define SHM_DEFAULT_PERMISSIONS 0660 +#define SHM_DEFAULT_SIZE 1024 + +typedef void (*shared_memory_init_fn)(void*); + +void* +get_or_create_shared_memory(uint64_t key, shared_memory_init_fn fn); + +void +free_shared_memory(void* key); + +#endif // EXT_SEMIAN_SYSV_SHARED_MEMORY_H diff --git a/ext/semian/types.h b/ext/semian/types.h index 3dade21d..a248151c 100644 --- a/ext/semian/types.h +++ b/ext/semian/types.h @@ -40,35 +40,29 @@ typedef struct { long wait_time; } semian_resource_t; -// Internal circuit breaker structure -typedef struct { - int sem_id; - uint64_t key; - char *strkey; - char *name; -} semian_circuit_breaker_t; - // Shared circuit breaker structure typedef struct { int successes; } semian_circuit_breaker_shared_t; -// Internal simple integer structure +// Internal circuit breaker structure typedef struct { uint64_t key; int sem_id; -} semian_simple_integer_t; + semian_circuit_breaker_shared_t* shmem; +} semian_circuit_breaker_t; // Shared simple integer structure typedef struct { int val; } semian_simple_integer_shared_t; -// Internal simple sliding window structure +// Internal simple integer structure typedef struct { uint64_t key; int sem_id; -} semian_simple_sliding_window_t; + semian_simple_integer_shared_t* shmem; +} semian_simple_integer_t; // Shared simple sliding window structure typedef struct { @@ -79,4 +73,11 @@ typedef struct { int data[SLIDING_WINDOW_MAX_SIZE]; } semian_simple_sliding_window_shared_t; +// Internal simple sliding window structure +typedef struct { + uint64_t key; + int sem_id; + semian_simple_sliding_window_shared_t* shmem; +} semian_simple_sliding_window_t; + #endif // SEMIAN_TYPES_H diff --git a/ext/semian/util.h b/ext/semian/util.h index 670b7d1e..0a126d8f 100644 --- a/ext/semian/util.h +++ b/ext/semian/util.h @@ -16,7 +16,7 @@ #define dprintf(fmt, ...) \ do { \ if (DEBUG_TEST) { \ - printf("[DEBUG] %s:%d" fmt "\n", __FILE__, __LINE__, ##__VA_ARGS__); \ + printf("[DEBUG] %s:%d - " fmt "\n", __FILE__, __LINE__, ##__VA_ARGS__); \ } \ } while (0) diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index 8225026c..321bd3d1 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -9,7 +9,7 @@ class CircuitBreaker #:nodoc: def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, implementation:, half_open_resource_timeout: nil) @name = name.to_sym - initialize_circuit_breaker(@name) + initialize_circuit_breaker("#{name}_circuit") @success_count_threshold = success_threshold @error_count_threshold = error_threshold diff --git a/lib/semian/circuit_breaker.rb.orig b/lib/semian/circuit_breaker.rb.orig index 5f812784..dea84572 100644 --- a/lib/semian/circuit_breaker.rb.orig +++ b/lib/semian/circuit_breaker.rb.orig @@ -6,10 +6,15 @@ module Semian attr_reader :name, :half_open_resource_timeout, :error_timeout, :state, :last_error +<<<<<<< HEAD def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, implementation:, half_open_resource_timeout: nil) @name = name.to_sym initialize_circuit_breaker(@name) +======= + def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, implementation:, half_open_resource_timeout: nil) + initialize_circuit_breaker("#{name}_circuit") +>>>>>>> More locks around shared data structures @success_count_threshold = success_threshold @error_count_threshold = error_threshold @@ -17,17 +22,12 @@ module Semian @exceptions = exceptions @half_open_resource_timeout = half_open_resource_timeout - @errors = implementation::SlidingWindow.new(max_size: @error_count_threshold) -<<<<<<< HEAD - @successes = implementation::Integer.new(name) - @state = implementation::State.new - - reset -======= + @errors = implementation::SlidingWindow.new("#{name}_window", max_size: @error_count_threshold) @successes = implementation::Integer.new("#{name}_successes") state_val = implementation::Integer.new("#{name}_state") @state = implementation::State.new(state_val) ->>>>>>> WIP: Move Simple::State to shared memory + + reset end def acquire(resource = nil, &block) @@ -139,7 +139,7 @@ module Semian def log_state_transition(new_state) return if @state.nil? || new_state == @state.value - str = "[#{self.class.name}] State transition from #{@state.value} to #{new_state}." + str = "[#{self.class.name}] State transition from #{@state} to #{new_state}." str << " success_count=#{@successes.value} error_count=#{@errors.size}" str << " success_count_threshold=#{@success_count_threshold} error_count_threshold=#{@error_count_threshold}" str << " error_timeout=#{@error_timeout} error_last_at=\"#{@errors.last}\"" From f251c8f340654fcc2e92416f74259045e8282011 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Mon, 17 Jun 2019 18:18:03 -0400 Subject: [PATCH 20/49] Reduce test flakiness with unique resource names --- test/circuit_breaker_test.rb | 7 ++++--- test/simple_integer_test.rb | 3 ++- test/simple_sliding_window_test.rb | 3 ++- test/simple_state_test.rb | 3 ++- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/test/circuit_breaker_test.rb b/test/circuit_breaker_test.rb index be4c01c9..3d51d79c 100644 --- a/test/circuit_breaker_test.rb +++ b/test/circuit_breaker_test.rb @@ -4,15 +4,16 @@ class TestCircuitBreaker < Minitest::Test include CircuitBreakerHelper def setup + id = Time.now.strftime('%H:%M:%S.%N') @strio = StringIO.new Semian.logger = Logger.new @strio begin - Semian.destroy(:testing) + Semian.destroy(id) rescue nil end - Semian.register(:testing, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) - @resource = Semian[:testing] + Semian.register(id, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) + @resource = Semian[id] end def test_acquire_yield_when_the_circuit_is_closed diff --git a/test/simple_integer_test.rb b/test/simple_integer_test.rb index e0d6da57..b1c541f2 100644 --- a/test/simple_integer_test.rb +++ b/test/simple_integer_test.rb @@ -2,7 +2,8 @@ class TestSimpleInteger < Minitest::Test def setup - @integer = ::Semian::ThreadSafe::Integer.new(:simple_integer) + id = Time.now.strftime('%H:%M:%S.%N') + @integer = ::Semian::ThreadSafe::Integer.new(id) end def teardown diff --git a/test/simple_sliding_window_test.rb b/test/simple_sliding_window_test.rb index d178e922..fb9e7be5 100644 --- a/test/simple_sliding_window_test.rb +++ b/test/simple_sliding_window_test.rb @@ -2,7 +2,8 @@ class TestSimpleSlidingWindow < Minitest::Test def setup - @sliding_window = ::Semian::ThreadSafe::SlidingWindow.new(:sliding_window_test, max_size: 6) + id = Time.now.strftime('%H:%M:%S.%N') + @sliding_window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 6) @sliding_window.clear end diff --git a/test/simple_state_test.rb b/test/simple_state_test.rb index 8dc3b4aa..9e8a9032 100644 --- a/test/simple_state_test.rb +++ b/test/simple_state_test.rb @@ -2,7 +2,8 @@ class TestSimpleEnum < Minitest::Test def setup - state_val = ::Semian::ThreadSafe::Integer.new(:test_simple_enum) + id = Time.now.strftime('%H:%M:%S.%N') + state_val = ::Semian::ThreadSafe::Integer.new(id) @state = ::Semian::ThreadSafe::State.new(state_val) end From 128788fb7509d5b9c7ac78baf7f5f24482fc10c2 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Tue, 18 Jun 2019 15:08:03 -0400 Subject: [PATCH 21/49] Grow/shrink the sliding window's max_size --- ext/semian/sliding_window.c | 98 ++++++++++++++++++++++++-- ext/semian/sliding_window.h | 1 + ext/semian/sysv_semaphores.c | 8 +-- ext/semian/sysv_semaphores.h | 4 +- ext/semian/tickets.c | 4 +- test/resource_test.rb | 1 + test/simple_sliding_window_test.rb | 107 ++++++++++++++++++++++++++++- 7 files changed, 204 insertions(+), 19 deletions(-) diff --git a/ext/semian/sliding_window.c b/ext/semian/sliding_window.c index 89dbf253..847bd374 100644 --- a/ext/semian/sliding_window.c +++ b/ext/semian/sliding_window.c @@ -60,6 +60,71 @@ check_max_size_arg(VALUE max_size) return retval; } +static VALUE +grow_window(semian_simple_sliding_window_shared_t* window, int new_max_size) +{ + if (new_max_size > SLIDING_WINDOW_MAX_SIZE) return Qnil; + + if (window->end > window->start) { + // Easy case - the window doesn't wrap around + if (window->length) window->end = window->start + window->length; + } else { + // Hard case - the window wraps, and data might need to move + int offset = new_max_size - window->max_size; + for (int i = offset - window->start - 1; i >= 0; --i) { + int srci = window->start + i; + int dsti = window->start + offset + i; + window->data[dsti] = window->data[srci]; + } + window->start += offset; + } + + window->max_size = new_max_size; + + return RB_INT2NUM(new_max_size); +} + +static void swap(int *a, int *b) { + int c = *a; + *a = *b; + *b = c; +} + +static VALUE +shrink_window(semian_simple_sliding_window_shared_t* window, int new_max_size) +{ + if (new_max_size > SLIDING_WINDOW_MAX_SIZE) return Qnil; + + int new_length = (new_max_size > window->length) ? window->length : new_max_size; + + if (window->end > window->start) { + // Easy case - the window doesn't wrap around + window->end = window->start + new_length; + } else { + // Hard case - the window wraps, so re-index the data + // Adapted from http://www.cplusplus.com/reference/algorithm/rotate/ + int first = 0; + int middle = window->start; + int last = window->max_size; + int next = middle; + while (first != next) { + swap(&window->data[first++], &window->data[next++]); + if (next == last) { + next = middle; + } else if (first == middle) { + middle = next; + } + } + window->start = 0; + window->end = new_length; + } + + window->max_size = new_max_size; + window->length = new_length; + + return RB_INT2NUM(new_max_size); +} + // Get the C object for a Ruby instance static semian_simple_sliding_window_t* get_object(VALUE self) @@ -81,6 +146,7 @@ Init_SlidingWindow() rb_define_alloc_func(cSlidingWindow, semian_simple_sliding_window_alloc); rb_define_method(cSlidingWindow, "initialize_sliding_window", semian_simple_sliding_window_initialize, 2); rb_define_method(cSlidingWindow, "size", semian_simple_sliding_window_size, 0); + rb_define_method(cSlidingWindow, "resize_to", semian_simple_sliding_window_resize_to, 1); rb_define_method(cSlidingWindow, "max_size", semian_simple_sliding_window_max_size, 0); rb_define_method(cSlidingWindow, "values", semian_simple_sliding_window_values, 0); rb_define_method(cSlidingWindow, "last", semian_simple_sliding_window_last, 0); @@ -115,11 +181,13 @@ semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_size) dprintf("Setting max_size for '%s' to %d", to_s(name), max_size_val); res->shmem->max_size = max_size_val; } else if (res->shmem->max_size != max_size_val) { - // TODO(michaelkipper): Figure out what do do in this case... - printf("Warning: Max size of %d is different than current value of %d", max_size_val, res->shmem->max_size); - // dprintf("max_size %d is different than %d", max_size_val, res->shmem->max_size); - // sem_meta_unlock(res->sem_id); - // rb_raise(rb_eArgError, "max_size was different"); + if (max_size_val > res->shmem->max_size) { + grow_window(res->shmem, max_size_val); + } else { + // TODO(michaelkipper): Figure out what do do in this case... + printf("Warning: Shrinking window from %d to %d", res->shmem->max_size, max_size_val); + shrink_window(res->shmem, max_size_val); + } } } sem_meta_unlock(res->sem_id); @@ -142,6 +210,26 @@ semian_simple_sliding_window_size(VALUE self) return retval; } +VALUE +semian_simple_sliding_window_resize_to(VALUE self, VALUE new_size) +{ + semian_simple_sliding_window_t *res = get_object(self); + VALUE retval = Qnil; + + int new_max_size = RB_NUM2INT(new_size); + sem_meta_lock(res->sem_id); + { + if (new_max_size > res->shmem->max_size) { + retval = grow_window(res->shmem, new_max_size); + } else if (new_max_size < res->shmem->max_size) { + retval = shrink_window(res->shmem, new_max_size); + } + } + sem_meta_unlock(res->sem_id); + + return retval; +} + VALUE semian_simple_sliding_window_max_size(VALUE self) { diff --git a/ext/semian/sliding_window.h b/ext/semian/sliding_window.h index 083df050..41f3ceba 100644 --- a/ext/semian/sliding_window.h +++ b/ext/semian/sliding_window.h @@ -9,6 +9,7 @@ void Init_SlidingWindow(); VALUE semian_simple_sliding_window_alloc(VALUE klass); VALUE semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_size); VALUE semian_simple_sliding_window_size(VALUE self); +VALUE semian_simple_sliding_window_resize_to(VALUE self, VALUE new_size); VALUE semian_simple_sliding_window_max_size(VALUE self); VALUE semian_simple_sliding_window_push(VALUE self, VALUE value); VALUE semian_simple_sliding_window_values(VALUE self); diff --git a/ext/semian/sysv_semaphores.c b/ext/semian/sysv_semaphores.c index 030ca75c..367af3d3 100644 --- a/ext/semian/sysv_semaphores.c +++ b/ext/semian/sysv_semaphores.c @@ -165,9 +165,7 @@ acquire_semaphore(void *p) semian_resource_t *res = (semian_resource_t *) p; res->error = 0; res->wait_time = -1; -#ifdef DEBUG - print_sem_vals(res->sem_id); -#endif + dprint_sem_vals(res->sem_id); struct timespec begin, end; int benchmark_result = clock_gettime(CLOCK_MONOTONIC, &begin); @@ -194,9 +192,7 @@ initialize_new_semaphore_values(int sem_id, long permissions) if (semctl(sem_id, 0, SETALL, init_vals) == -1) { raise_semian_syscall_error("semctl()", errno); } -#ifdef DEBUG - print_sem_vals(sem_id); -#endif + dprint_sem_vals(sem_id); } static int diff --git a/ext/semian/sysv_semaphores.h b/ext/semian/sysv_semaphores.h index 9d4e2203..d680fd0f 100644 --- a/ext/semian/sysv_semaphores.h +++ b/ext/semian/sysv_semaphores.h @@ -115,9 +115,8 @@ acquire_semaphore_without_gvl(void *p); int initialize_single_semaphore(uint64_t key, long permissions); -#ifdef DEBUG static inline void -print_sem_vals(int sem_id) +dprint_sem_vals(int sem_id) { dprintf("sem_id: %d, lock: %d, tickets: %d configured: %d, registered workers %d", sem_id, @@ -127,6 +126,5 @@ print_sem_vals(int sem_id) get_sem_val(sem_id, SI_SEM_REGISTERED_WORKERS) ); } -#endif #endif // SEMIAN_SEMSET_H diff --git a/ext/semian/tickets.c b/ext/semian/tickets.c index 369f3058..08d2dbe0 100644 --- a/ext/semian/tickets.c +++ b/ext/semian/tickets.c @@ -49,9 +49,7 @@ update_ticket_count(int sem_id, int tickets) delta = tickets - get_sem_val(sem_id, SI_SEM_CONFIGURED_TICKETS); -#ifdef DEBUG - print_sem_vals(sem_id); -#endif + dprint_sem_vals(sem_id); if (perform_semop(sem_id, SI_SEM_TICKETS, delta, 0, &ts) == -1) { if (delta < 0 && errno == EAGAIN) { rb_raise(eTimeout, "timeout while trying to update ticket count"); diff --git a/test/resource_test.rb b/test/resource_test.rb index aa624a42..ca4a3b12 100644 --- a/test/resource_test.rb +++ b/test/resource_test.rb @@ -8,6 +8,7 @@ class TestResource < Minitest::Test EPSILON = 0.1 def setup + @workers = [] Semian.destroy(:testing) rescue nil diff --git a/test/simple_sliding_window_test.rb b/test/simple_sliding_window_test.rb index fb9e7be5..65cb9b77 100644 --- a/test/simple_sliding_window_test.rb +++ b/test/simple_sliding_window_test.rb @@ -49,10 +49,113 @@ def resize_to_less_than_1_raises end end + def test_resize_to_simple + id = Time.now.strftime('%H:%M:%S.%N') + window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) + window << 0 << 1 + assert_sliding_window(window, [0, 1], 4) + window.resize_to(8) + assert_sliding_window(window, [0, 1], 8) + window << 2 << 3 << 4 << 5 + assert_sliding_window(window, [0, 1, 2, 3, 4, 5], 8) + end + + def test_resize_to_simple_full + id = Time.now.strftime('%H:%M:%S.%N') + window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) + window << 0 << 1 << 2 << 3 + assert_sliding_window(window, [0, 1, 2, 3], 4) + window.resize_to(8) + assert_sliding_window(window, [0, 1, 2, 3], 8) + window << 4 << 5 + assert_sliding_window(window, [0, 1, 2, 3, 4, 5], 8) + end + + def test_resize_to_simple_floating + id = Time.now.strftime('%H:%M:%S.%N') + window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) + window << 0 << 1 << 2 << 3 + assert_sliding_window(window, [0, 1, 2, 3], 4) + window.reject! { |val| val < 2 } + assert_sliding_window(window, [2, 3], 4) + window.resize_to(8) + assert_sliding_window(window, [2, 3], 8) + window << 4 << 5 + assert_sliding_window(window, [2, 3, 4, 5], 8) + end + + def test_resize_to_hard + id = Time.now.strftime('%H:%M:%S.%N') + window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) + window << 0 << 1 << 2 << 3 << 4 << 5 + assert_sliding_window(window, [2, 3, 4, 5], 4) + window.resize_to(8) + assert_sliding_window(window, [2, 3, 4, 5], 8) + window << 6 << 7 + assert_sliding_window(window, [2, 3, 4, 5, 6, 7], 8) + window << 8 << 9 + assert_sliding_window(window, [2, 3, 4, 5, 6, 7, 8, 9], 8) + end + + def test_resize_to_shrink_simple + id = Time.now.strftime('%H:%M:%S.%N') + window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) + window << 0 << 1 + assert_sliding_window(window, [0, 1], 4) + window.resize_to(2) + assert_sliding_window(window, [0, 1], 2) + end + + def test_resize_to_shrink_simple_full + id = Time.now.strftime('%H:%M:%S.%N') + window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) + window << 0 << 1 << 2 << 3 + assert_sliding_window(window, [0, 1, 2, 3], 4) + window.resize_to(2) + assert_sliding_window(window, [0, 1], 2) + end + + def test_resize_to_shrink_hard + id = Time.now.strftime('%H:%M:%S.%N') + window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) + window << 0 << 1 << 2 << 3 << 4 << 5 + assert_sliding_window(window, [2, 3, 4, 5], 4) + window.resize_to(2) + assert_sliding_window(window, [2, 3], 2) + end + + def test_resize_to_shrink_all_index + 4.times do |offset| + id = Time.now.strftime('%H:%M:%S.%N-#{offset}') + window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) + offset.times { window << offset } + window << 0 << 1 << 2 << 3 + assert_sliding_window(window, [0, 1, 2, 3], 4) + window.resize_to(2) + assert_sliding_window(window, [0, 1], 2) + end + end + + def test_resize_to_grow_all_index + 4.times do |offset| + id = Time.now.strftime('%H:%M:%S.%N-#{offset}') + window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) + offset.times { window << offset } + window << 0 << 1 << 2 << 3 + assert_sliding_window(window, [0, 1, 2, 3], 4) + window.resize_to(8) + assert_sliding_window(window, [0, 1, 2, 3], 8) + window << 4 << 5 << 6 << 7 + assert_sliding_window(window, [0, 1, 2, 3, 4, 5, 6, 7], 8) + window << 8 << 9 + assert_sliding_window(window, [2, 3, 4, 5, 6, 7, 8, 9], 8) + end + end + private def assert_sliding_window(sliding_window, array, max_size) - assert_equal(array, sliding_window.values) - assert_equal(max_size, sliding_window.max_size) + assert_equal(array, sliding_window.values, "Window contents were different") + assert_equal(max_size, sliding_window.max_size, "Window max_size was not equal") end end From 5bb4d65159d96781f780cde3c093bd1114a0f36e Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Tue, 18 Jun 2019 19:48:37 -0400 Subject: [PATCH 22/49] Ignore CLion project files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 95d00207..77e5324f 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ nohup.out # IntelliJ/RubyMine/CLion project files .idea +CMakeLists.txt +cmake-build-debug From f6bcf7f900e4f4eff27ac808c2fe0cb244aabf25 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Tue, 18 Jun 2019 19:49:06 -0400 Subject: [PATCH 23/49] WIP: Registered worker based scaling factor for error_threshold --- ext/semian/circuit_breaker.c | 10 +- ext/semian/resource.c | 18 +-- ext/semian/sliding_window.c | 100 +++++++++++---- ext/semian/types.h | 3 +- lib/semian/circuit_breaker.rb | 8 +- lib/semian/circuit_breaker.rb.orig | 171 ------------------------- test/circuit_breaker_test.rb | 1 + test/helpers/circuit_breaker_helper.rb | 5 +- test/simple_sliding_window_test.rb | 25 +++- 9 files changed, 124 insertions(+), 217 deletions(-) delete mode 100644 lib/semian/circuit_breaker.rb.orig diff --git a/ext/semian/circuit_breaker.c b/ext/semian/circuit_breaker.c index f689cabe..f36034a6 100644 --- a/ext/semian/circuit_breaker.c +++ b/ext/semian/circuit_breaker.c @@ -37,7 +37,7 @@ Init_CircuitBreaker() { VALUE cCircuitBreaker = rb_const_get(cSemian, rb_intern("CircuitBreaker")); rb_define_alloc_func(cCircuitBreaker, semian_circuit_breaker_alloc); - rb_define_method(cCircuitBreaker, "initialize_circuit_breaker", semian_circuit_breaker_initialize, 1); + rb_define_method(cCircuitBreaker, "initialize_circuit_breaker", semian_circuit_breaker_initialize, 2); } VALUE @@ -53,9 +53,13 @@ semian_circuit_breaker_initialize(VALUE self, VALUE name) { semian_circuit_breaker_t *res = NULL; TypedData_Get_Struct(self, semian_circuit_breaker_t, &semian_circuit_breaker_type, res); - res->key = generate_key(to_s(name)); - dprintf("Initializing circuit breaker '%s' (key: %lu)", to_s(name), res->key); + char buffer[1024]; + strcpy(buffer, to_s(name)); + strcat(buffer, "_circuit_breaker"); + res->key = generate_key(buffer); + + dprintf("Initializing circuit breaker '%s' (key: %lu)", buffer, res->key); res->sem_id = initialize_single_semaphore(res->key, SEM_DEFAULT_PERMISSIONS); res->shmem = get_or_create_shared_memory(res->key, NULL); diff --git a/ext/semian/resource.c b/ext/semian/resource.c index feed7a2d..52144cfb 100644 --- a/ext/semian/resource.c +++ b/ext/semian/resource.c @@ -201,22 +201,16 @@ semian_resource_key(VALUE self) VALUE semian_resource_initialize(VALUE self, VALUE id, VALUE tickets, VALUE quota, VALUE permissions, VALUE default_timeout) { - long c_permissions; - double c_timeout; - double c_quota; - int c_tickets; - semian_resource_t *res = NULL; - const char *c_id_str = NULL; - // Check and cast arguments check_tickets_xor_quota_arg(tickets, quota); - c_quota = check_quota_arg(quota); - c_tickets = check_tickets_arg(tickets); - c_permissions = check_permissions_arg(permissions); - c_id_str = check_id_arg(id); - c_timeout = check_default_timeout_arg(default_timeout); + double c_quota = check_quota_arg(quota); + int c_tickets = check_tickets_arg(tickets); + long c_permissions = check_permissions_arg(permissions); + const char *c_id_str = check_id_arg(id); + double c_timeout = check_default_timeout_arg(default_timeout); // Build semian resource structure + semian_resource_t *res = NULL; TypedData_Get_Struct(self, semian_resource_t, &semian_resource_type, res); // Populate struct fields diff --git a/ext/semian/sliding_window.c b/ext/semian/sliding_window.c index 847bd374..01ce3a3d 100644 --- a/ext/semian/sliding_window.c +++ b/ext/semian/sliding_window.c @@ -65,9 +65,12 @@ grow_window(semian_simple_sliding_window_shared_t* window, int new_max_size) { if (new_max_size > SLIDING_WINDOW_MAX_SIZE) return Qnil; - if (window->end > window->start) { + if (window->length == 0) { + window->start = 0; + window->end = 0; + } else if (window->end > window->start) { // Easy case - the window doesn't wrap around - if (window->length) window->end = window->start + window->length; + window->end = window->start + window->length; } else { // Hard case - the window wraps, and data might need to move int offset = new_max_size - window->max_size; @@ -97,7 +100,11 @@ shrink_window(semian_simple_sliding_window_shared_t* window, int new_max_size) int new_length = (new_max_size > window->length) ? window->length : new_max_size; - if (window->end > window->start) { + dprintf("Shrinking window - start:%d end:%d length:%d max_size:%d", window->start, window->end, window->length, window->max_size); + if (window->length == 0) { + window->start = 0; + window->end = 0; + } else if (window->end > window->start) { // Easy case - the window doesn't wrap around window->end = window->start + new_length; } else { @@ -125,6 +132,24 @@ shrink_window(semian_simple_sliding_window_shared_t* window, int new_max_size) return RB_INT2NUM(new_max_size); } +static VALUE +resize_window(semian_simple_sliding_window_shared_t* window, int new_max_size) +{ + if (new_max_size > SLIDING_WINDOW_MAX_SIZE) return Qnil; + + if (window->max_size < new_max_size) { + dprintf("Growing window to %d", new_max_size); + return grow_window(window, new_max_size); + } else if (window->max_size > new_max_size) { + dprintf("Shrinking window to %d", new_max_size); + return shrink_window(window, new_max_size); + } else { + dprintf("Not re-sizing window"); + } + + return Qnil; +} + // Get the C object for a Ruby instance static semian_simple_sliding_window_t* get_object(VALUE self) @@ -151,6 +176,7 @@ Init_SlidingWindow() rb_define_method(cSlidingWindow, "values", semian_simple_sliding_window_values, 0); rb_define_method(cSlidingWindow, "last", semian_simple_sliding_window_last, 0); rb_define_method(cSlidingWindow, "<<", semian_simple_sliding_window_push, 1); + rb_define_method(cSlidingWindow, "clear", semian_simple_sliding_window_clear, 0); rb_define_method(cSlidingWindow, "destroy", semian_simple_sliding_window_clear, 0); rb_define_method(cSlidingWindow, "reject!", semian_simple_sliding_window_reject, 0); } @@ -163,32 +189,53 @@ semian_simple_sliding_window_alloc(VALUE klass) return obj; } +static int +get_number_of_registered_workers(semian_simple_sliding_window_t* res) +{ + int sem_id = semget(res->parent_key, SI_NUM_SEMAPHORES, SEM_DEFAULT_PERMISSIONS); + if (sem_id == -1) { + dprintf("Warning: Could not get semaphore for key=%d", res->parent_key); + return 1; + } + + int retval = semctl(sem_id, SI_SEM_REGISTERED_WORKERS, GETVAL); + if (retval == -1) { + dprintf("Warning: Could not get SI_SEM_REGISTERED_WORKERS for sem_id=%d", sem_id); + return 1; + } + + return retval; +} + +static int max(int a, int b) { + return a > b ? a : b; +} + VALUE semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_size) { semian_simple_sliding_window_t *res = get_object(self); - res->key = generate_key(to_s(name)); - dprintf("Initializing simple sliding window '%s' (key: %lu)", to_s(name), res->key); + char buffer[1024]; + strcpy(buffer, to_s(name)); + strcat(buffer, "_sliding_window"); + res->key = generate_key(buffer); + + // Store the parent key, not the parent sem_id, since it might not exist yet. + res->parent_key = generate_key(to_s(name)); + + dprintf("Initializing simple sliding window '%s' (key: %lu)", buffer, res->key); res->sem_id = initialize_single_semaphore(res->key, SEM_DEFAULT_PERMISSIONS); res->shmem = get_or_create_shared_memory(res->key, init_fn); - - int max_size_val = check_max_size_arg(max_size); + res->error_threshold = check_max_size_arg(max_size); sem_meta_lock(res->sem_id); { - if (res->shmem->max_size == 0) { - dprintf("Setting max_size for '%s' to %d", to_s(name), max_size_val); - res->shmem->max_size = max_size_val; - } else if (res->shmem->max_size != max_size_val) { - if (max_size_val > res->shmem->max_size) { - grow_window(res->shmem, max_size_val); - } else { - // TODO(michaelkipper): Figure out what do do in this case... - printf("Warning: Shrinking window from %d to %d", res->shmem->max_size, max_size_val); - shrink_window(res->shmem, max_size_val); - } - } + int workers = get_number_of_registered_workers(res); + float scale_factor = (workers > 1) ? 0.2 : 1.0; // TODO: Parameterize + int error_threshold = max(res->error_threshold, (int) ceil(workers * scale_factor * res->error_threshold)); + + resize_window(res->shmem, error_threshold); } sem_meta_unlock(res->sem_id); @@ -219,11 +266,7 @@ semian_simple_sliding_window_resize_to(VALUE self, VALUE new_size) int new_max_size = RB_NUM2INT(new_size); sem_meta_lock(res->sem_id); { - if (new_max_size > res->shmem->max_size) { - retval = grow_window(res->shmem, new_max_size); - } else if (new_max_size < res->shmem->max_size) { - retval = shrink_window(res->shmem, new_max_size); - } + retval = resize_window(res->shmem, new_max_size); } sem_meta_unlock(res->sem_id); @@ -288,6 +331,7 @@ semian_simple_sliding_window_clear(VALUE self) sem_meta_lock(res->sem_id); { + dprintf("Clearing sliding window"); res->shmem->length = 0; res->shmem->start = 0; res->shmem->end = 0; @@ -309,6 +353,7 @@ semian_simple_sliding_window_reject(VALUE self) // Store these values because we're going to be modifying the buffer. int start = res->shmem->start; int length = res->shmem->length; + dprintf("reject! - start:%d end:%d length:%d max_size:%d", res->shmem->start, res->shmem->end, res->shmem->length, res->shmem->max_size); int cleared = 0; for (int i = 0; i < length; ++i) { @@ -337,14 +382,17 @@ semian_simple_sliding_window_push(VALUE self, VALUE value) sem_meta_lock(res->sem_id); { + dprintf("Before: start:%d end:%d length:%d max_size:%d", res->shmem->start, res->shmem->end, res->shmem->length, res->shmem->max_size); + // If the window is full, make room by popping off the front. if (res->shmem->length == res->shmem->max_size) { res->shmem->length--; res->shmem->start = (res->shmem->start + 1) % res->shmem->max_size; } - const int index = res->shmem->end; + // Push onto the back of the window. res->shmem->length++; - res->shmem->data[index] = RB_NUM2INT(value); + res->shmem->data[res->shmem->end] = RB_NUM2INT(value); + dprintf("Pushed %d onto data[%d] (length %d)", RB_NUM2INT(value), res->shmem->end, res->shmem->length); res->shmem->end = (res->shmem->end + 1) % res->shmem->max_size; } sem_meta_unlock(res->sem_id); diff --git a/ext/semian/types.h b/ext/semian/types.h index a248151c..4bca785a 100644 --- a/ext/semian/types.h +++ b/ext/semian/types.h @@ -42,7 +42,6 @@ typedef struct { // Shared circuit breaker structure typedef struct { - int successes; } semian_circuit_breaker_shared_t; // Internal circuit breaker structure @@ -77,6 +76,8 @@ typedef struct { typedef struct { uint64_t key; int sem_id; + uint64_t parent_key; + int error_threshold; semian_simple_sliding_window_shared_t* shmem; } semian_simple_sliding_window_t; diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index 321bd3d1..c5ca5105 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -9,7 +9,7 @@ class CircuitBreaker #:nodoc: def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, implementation:, half_open_resource_timeout: nil) @name = name.to_sym - initialize_circuit_breaker("#{name}_circuit") + initialize_circuit_breaker(name, error_threshold) @success_count_threshold = success_threshold @error_count_threshold = error_threshold @@ -17,7 +17,7 @@ def initialize(name, exceptions:, success_threshold:, error_threshold:, @exceptions = exceptions @half_open_resource_timeout = half_open_resource_timeout - @errors = implementation::SlidingWindow.new("#{name}_window", max_size: @error_count_threshold) + @errors = implementation::SlidingWindow.new(name, max_size: @error_count_threshold) @successes = implementation::Integer.new("#{name}_successes") state_val = implementation::Integer.new("#{name}_state") @state = implementation::State.new(state_val) @@ -162,5 +162,9 @@ def maybe_with_half_open_resource_timeout(resource, &block) result end + + def to_s + "" + end end end diff --git a/lib/semian/circuit_breaker.rb.orig b/lib/semian/circuit_breaker.rb.orig deleted file mode 100644 index dea84572..00000000 --- a/lib/semian/circuit_breaker.rb.orig +++ /dev/null @@ -1,171 +0,0 @@ -module Semian - class CircuitBreaker #:nodoc: - extend Forwardable - - def_delegators :@state, :closed?, :open?, :half_open? - - attr_reader :name, :half_open_resource_timeout, :error_timeout, :state, :last_error - -<<<<<<< HEAD - def initialize(name, exceptions:, success_threshold:, error_threshold:, - error_timeout:, implementation:, half_open_resource_timeout: nil) - @name = name.to_sym - initialize_circuit_breaker(@name) -======= - def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, implementation:, half_open_resource_timeout: nil) - initialize_circuit_breaker("#{name}_circuit") ->>>>>>> More locks around shared data structures - - @success_count_threshold = success_threshold - @error_count_threshold = error_threshold - @error_timeout = error_timeout - @exceptions = exceptions - @half_open_resource_timeout = half_open_resource_timeout - - @errors = implementation::SlidingWindow.new("#{name}_window", max_size: @error_count_threshold) - @successes = implementation::Integer.new("#{name}_successes") - state_val = implementation::Integer.new("#{name}_state") - @state = implementation::State.new(state_val) - - reset - end - - def acquire(resource = nil, &block) - return yield if disabled? - transition_to_half_open if transition_to_half_open? - - raise OpenCircuitError unless request_allowed? - - result = nil - begin - result = maybe_with_half_open_resource_timeout(resource, &block) - rescue *@exceptions => error - if !error.respond_to?(:marks_semian_circuits?) || error.marks_semian_circuits? - mark_failed(error) - end - raise error - else - mark_success - end - result - end - - def transition_to_half_open? - open? && error_timeout_expired? && !half_open? - end - - def request_allowed? - closed? || half_open? || transition_to_half_open? - end - - def mark_failed(error) - push_error(error) - push_time(@errors) - if closed? - transition_to_open if error_threshold_reached? - elsif half_open? - transition_to_open - end - end - - def mark_success - return unless half_open? - @successes.increment(1) - transition_to_close if success_threshold_reached? - end - - def reset - @errors.clear - @successes.reset - transition_to_close - end - - def destroy - @errors.destroy - @successes.destroy - @state.destroy - end - - def in_use? - return false if error_timeout_expired? - @errors.size > 0 - end - - private - - def transition_to_close - notify_state_transition(:closed) - log_state_transition(:closed) - @state.close! - @errors.clear - end - - def transition_to_open - notify_state_transition(:open) - log_state_transition(:open) - @state.open! - end - - def transition_to_half_open - notify_state_transition(:half_open) - log_state_transition(:half_open) - @state.half_open! - @successes.reset - end - - def success_threshold_reached? - @successes.value >= @success_count_threshold - end - - def error_threshold_reached? - @errors.size == @error_count_threshold - end - - def error_timeout_expired? - last_error_time = @errors.last - return false unless last_error_time - Time.at(last_error_time) + @error_timeout < Time.now - end - - def push_error(error) - @last_error = error - end - - def push_time(window, time: Time.now) - window.reject! { |err_time| err_time + @error_timeout < time.to_i } - window << time.to_i - end - - def log_state_transition(new_state) - return if @state.nil? || new_state == @state.value - - str = "[#{self.class.name}] State transition from #{@state} to #{new_state}." - str << " success_count=#{@successes.value} error_count=#{@errors.size}" - str << " success_count_threshold=#{@success_count_threshold} error_count_threshold=#{@error_count_threshold}" - str << " error_timeout=#{@error_timeout} error_last_at=\"#{@errors.last}\"" - str << " name=\"#{@name}\"" - Semian.logger.info(str) - end - - def notify_state_transition(new_state) - Semian.notify(:state_change, self, nil, nil, state: new_state) - end - - def disabled? - ENV['SEMIAN_CIRCUIT_BREAKER_DISABLED'] || ENV['SEMIAN_DISABLED'] - end - - def maybe_with_half_open_resource_timeout(resource, &block) - result = - if half_open? && @half_open_resource_timeout && resource.respond_to?(:with_resource_timeout) - resource.with_resource_timeout(@half_open_resource_timeout) do - block.call - end - else - block.call - end - - result - end - end -end diff --git a/test/circuit_breaker_test.rb b/test/circuit_breaker_test.rb index 3d51d79c..eacd1776 100644 --- a/test/circuit_breaker_test.rb +++ b/test/circuit_breaker_test.rb @@ -25,6 +25,7 @@ def test_acquire_yield_when_the_circuit_is_closed def test_acquire_raises_circuit_open_error_when_the_circuit_is_open open_circuit! assert_raises Semian::OpenCircuitError do + puts "test: Acquiring resource #{@resource.circuit_breaker}" @resource.acquire { 1 + 1 } end assert_match(/State transition from closed to open/, @strio.string) diff --git a/test/helpers/circuit_breaker_helper.rb b/test/helpers/circuit_breaker_helper.rb index 7d86ae0d..764effa7 100644 --- a/test/helpers/circuit_breaker_helper.rb +++ b/test/helpers/circuit_breaker_helper.rb @@ -14,7 +14,10 @@ def half_open_cicuit!(resource = @resource, backwards_time_travel = 10) end def trigger_error!(resource = @resource, error = SomeError) - resource.acquire { raise error } + resource.acquire do + puts "Triggering error" + raise error + end rescue error end diff --git a/test/simple_sliding_window_test.rb b/test/simple_sliding_window_test.rb index 65cb9b77..d420461f 100644 --- a/test/simple_sliding_window_test.rb +++ b/test/simple_sliding_window_test.rb @@ -11,6 +11,24 @@ def teardown @sliding_window.destroy end + def test_clear + id = Time.now.strftime('%H:%M:%S.%N') + window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 6) + window << 1 << 2 << 3 + assert_equal(3, window.size) + window.clear + assert_equal(0, window.size) + end + + def test_destroy + id = Time.now.strftime('%H:%M:%S.%N') + window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 6) + window << 1 << 2 << 3 + assert_equal(3, window.size) + window.destroy + assert_equal(0, window.size) + end + def test_sliding_window_push assert_equal(0, @sliding_window.size) @sliding_window << 1 @@ -43,12 +61,17 @@ def test_sliding_window_reject_failure end end - def resize_to_less_than_1_raises + def test_resize_to_less_than_1_raises assert_raises ArgumentError do @sliding_window.resize_to 0 end end + def test_resize_to_1_works + assert_equal(0, @sliding_window.size) + @sliding_window.resize_to 1 + end + def test_resize_to_simple id = Time.now.strftime('%H:%M:%S.%N') window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) From 3498109b5ccf45464b2859cbad1522ad9a7476bc Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Wed, 19 Jun 2019 12:08:13 -0400 Subject: [PATCH 24/49] Fixed compile error --- ext/semian/sliding_window.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/semian/sliding_window.c b/ext/semian/sliding_window.c index 01ce3a3d..6def16bb 100644 --- a/ext/semian/sliding_window.c +++ b/ext/semian/sliding_window.c @@ -194,7 +194,7 @@ get_number_of_registered_workers(semian_simple_sliding_window_t* res) { int sem_id = semget(res->parent_key, SI_NUM_SEMAPHORES, SEM_DEFAULT_PERMISSIONS); if (sem_id == -1) { - dprintf("Warning: Could not get semaphore for key=%d", res->parent_key); + dprintf("Warning: Could not get semaphore for key=%lu", res->parent_key); return 1; } From e4631b0d76d29cddf728e8c416f56f9853c02578 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Mon, 24 Jun 2019 09:10:56 -0400 Subject: [PATCH 25/49] Removed spammy debug message --- test/simple_integer_test.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/simple_integer_test.rb b/test/simple_integer_test.rb index b1c541f2..f925c86b 100644 --- a/test/simple_integer_test.rb +++ b/test/simple_integer_test.rb @@ -56,8 +56,6 @@ def test_increment_race end exit_codes = Process.waitall.map { |_, status| status.exitstatus } # No two processes should exit with the same exit code - duplicate_values = exit_codes.group_by { |i| i }.select { |_, v| v.size > 1 } - puts "Duplicate values: #{duplicate_values}" unless duplicate_values.empty? assert_equal(process_count, exit_codes.uniq.length) end end From effaea61c7ca18e3b5b024f588490435d8d3e6df Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Mon, 24 Jun 2019 09:11:32 -0400 Subject: [PATCH 26/49] Made the default for SEMIAN_CIRCUIT_BREAKER_IMPL to be worker/Ruby based --- ext/semian/semian.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ext/semian/semian.c b/ext/semian/semian.c index b21a7f64..7a2a88f8 100644 --- a/ext/semian/semian.c +++ b/ext/semian/semian.c @@ -77,9 +77,19 @@ void Init_semian() static int use_c_circuits() { char *circuit_impl = getenv("SEMIAN_CIRCUIT_BREAKER_IMPL"); - if (circuit_impl == NULL || strcmp(circuit_impl, "ruby") != 0) { - return 1; - } else { + if (circuit_impl == NULL) { + fprintf(stderr, "Warning: Defaulting to worker-based circuit breaker implementation\n"); return 0; + } else { + if (!strcmp(circuit_impl, "ruby") || !strcmp(circuit_impl, "worker")) { + return 0; + } else if (!strcmp(circuit_impl, "c") || !strcmp(circuit_impl, "host")) { + return 1; + } else { + fprintf(stderr, "Warning: Unknown circuit breaker implementation: '%s'\n", circuit_impl); + return 0; + } } + + rb_raise(rb_eArgError, "Unknown circuit breaker implementation"); } From bff92255d766ba45b221723fc0b77988bbbf56dc Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Mon, 24 Jun 2019 09:11:52 -0400 Subject: [PATCH 27/49] Run both worker/host based circuit breaker implementations in CI --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 46be05f4..70c20b82 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,8 @@ matrix: - env: GEMFILE=gemfiles/hiredis-0-6.gemfile - env: GEMFILE=gemfiles/mysql2-0-4-10.gemfile - env: GEMFILE=gemfiles/mysql2-0-5-0.gemfile - - env: GEMFILE=Gemfile + - env: GEMFILE=Gemfile SEMIAN_CIRCUIT_BREAKER_IMPL=worker + - env: GEMFILE=Gemfile SEMIAN_CIRCUIT_BREAKER_IMPL=host script: - docker build --build-arg BUNDLE_GEMFILE="$GEMFILE" -t shopify/semian-ci:latest -f dockerfiles/semian-ci . From 0ae9c55f644f042a8c2f466707b31297ddc23f18 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Tue, 25 Jun 2019 10:34:20 -0400 Subject: [PATCH 28/49] Allow SEMIAN_DEBUG to enable debug messages --- ext/semian/util.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/semian/util.h b/ext/semian/util.h index 0a126d8f..242a4a09 100644 --- a/ext/semian/util.h +++ b/ext/semian/util.h @@ -7,7 +7,7 @@ #include #include -#ifdef DEBUG +#if defined(DEBUG) || defined(SEMIAN_DEBUG) # define DEBUG_TEST 1 #else # define DEBUG_TEST 0 From 26aa0dd3b1b906391be499d5195c08d0001f2c82 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Tue, 25 Jun 2019 12:08:36 -0400 Subject: [PATCH 29/49] Ignore log files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 77e5324f..1bc40cbd 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ /tmp /pkg *.gem +*.log /html/ Gemfile.lock vendor/ From 43f6a57e3ecc027a10149d2b59de32276dd8fd99 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Tue, 25 Jun 2019 12:26:00 -0400 Subject: [PATCH 30/49] Review comments --- README.md | 27 +++++++++++++++++++++++++++ ext/semian/semian.c | 4 ++-- test/simple_integer_test.rb | 28 ++++++++++++++++------------ test/simple_sliding_window_test.rb | 1 + 4 files changed, 46 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 8a9159e1..32da4895 100644 --- a/README.md +++ b/README.md @@ -448,6 +448,33 @@ There are three configuration parameters for circuit breakers in Semian: * **success_threshold**. The amount of successes on the circuit until closing it again, that is to start accepting all requests to the circuit. * **half_open_resource_timeout**. Timeout for the resource in seconds when the circuit is half-open (only supported for MySQL and Net::HTTP). +* **scale_factor**. When using [host-based circuits](#host-based-circuits), the + scaling factor to determine how to scale `error_threshold * num_workers` to + achieve faster circuit opens. + +#### Host-Based Circuits + +On systems with [SysV support][sysv], we can share circuit error information +between processes on a server. This means that those processes can effectively +share information between each other about resource health, leading to faster, +more efficient opening of circuits. + +As an example, imagine a system with _N_ processes on a host with an error +threshold of _E_, and a client timeout of _T_. By default, the system needs to +see _N * E_ errors to open the circuit. We can reduce this by using the +`scale_factor` configuration parameter. If we set `scale_factor` to _1 / N_, +the total number of errors we'd need to see server-wide is still _E_. In +this configuration, we can reduce the time-to-open for a circuit from _E * T_ +to simply _T_ (provided that _N_ is greater than _E_). + +You should run a simulation with your workloads to determine an efficient +scaling factor that will produce a time-to-open reduction but isn't too +sensitive. + +The circuit breaker implementation is based on the environment variable +`SEMIAN_CIRCUIT_BREAKER_IMPL`. Set it to `worker` to disable sharing circuit +state and `host` to enable host-based circuits. The default is `worker` for +backward compatibility. ### Bulkheading diff --git a/ext/semian/semian.c b/ext/semian/semian.c index 7a2a88f8..34a0271c 100644 --- a/ext/semian/semian.c +++ b/ext/semian/semian.c @@ -81,9 +81,9 @@ use_c_circuits() { fprintf(stderr, "Warning: Defaulting to worker-based circuit breaker implementation\n"); return 0; } else { - if (!strcmp(circuit_impl, "ruby") || !strcmp(circuit_impl, "worker")) { + if (!strcmp(circuit_impl, "worker")) { return 0; - } else if (!strcmp(circuit_impl, "c") || !strcmp(circuit_impl, "host")) { + } else if (!strcmp(circuit_impl, "host")) { return 1; } else { fprintf(stderr, "Warning: Unknown circuit breaker implementation: '%s'\n", circuit_impl); diff --git a/test/simple_integer_test.rb b/test/simple_integer_test.rb index f925c86b..925c425f 100644 --- a/test/simple_integer_test.rb +++ b/test/simple_integer_test.rb @@ -43,20 +43,24 @@ def test_reset assert_equal(0, @integer.value) end - if ENV['SEMIAN_CIRCUIT_BREAKER_IMPL'] != 'ruby' - # Without locks, this only passes around 1 in every 5 runs - def test_increment_race - process_count = 255 - 100.times do - process_count.times do - fork do - value = @integer.increment(1) - exit!(value) - end + # Without locks, this only passes around 1 in every 5 runs + def test_increment_race + process_count = 255 + 100.times do + process_count.times do + fork do + value = @integer.increment(1) + exit!(value) end - exit_codes = Process.waitall.map { |_, status| status.exitstatus } - # No two processes should exit with the same exit code + end + exit_codes = Process.waitall.map { |_, status| status.exitstatus } + + if ENV['SEMIAN_CIRCUIT_BREAKER_IMPL'] == 'host' + # Host-based circuits: No two processes should exit with the same exit code assert_equal(process_count, exit_codes.uniq.length) + else + # Worker-based circuits: All the processes should exit with the same code + assert_equal(1, exit_codes.uniq.length) end end end diff --git a/test/simple_sliding_window_test.rb b/test/simple_sliding_window_test.rb index d420461f..35c55a1d 100644 --- a/test/simple_sliding_window_test.rb +++ b/test/simple_sliding_window_test.rb @@ -52,6 +52,7 @@ def test_sliding_window_reject end def test_sliding_window_reject_failure + skip if ENV['SEMIAN_CIRCUIT_BREAKER_IMPL'] == 'worker' @sliding_window << 0 << 1 << 2 << 3 << 4 << 5 << 6 << 7 assert_equal(6, @sliding_window.size) assert_sliding_window(@sliding_window, [2, 3, 4, 5, 6, 7], 6) From dafe3629ac856daa09126c6e44aab771e2243ba6 Mon Sep 17 00:00:00 2001 From: Aditya Sharma Date: Tue, 25 Jun 2019 19:50:08 -0400 Subject: [PATCH 31/49] new build matrix --- .travis.yml | 22 ++++++++++++++-------- dockerfiles/semian-ci | 3 +++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 70c20b82..733354bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,16 +13,22 @@ before_install: services: - docker -matrix: - include: - - env: GEMFILE=gemfiles/hiredis-0-6.gemfile - - env: GEMFILE=gemfiles/mysql2-0-4-10.gemfile - - env: GEMFILE=gemfiles/mysql2-0-5-0.gemfile - - env: GEMFILE=Gemfile SEMIAN_CIRCUIT_BREAKER_IMPL=worker - - env: GEMFILE=Gemfile SEMIAN_CIRCUIT_BREAKER_IMPL=host +gemfile: + - Gemfile + - gemfiles/hiredis-0-6.gemfile + - gemfiles/mysql2-0-4-10.gemfile + - gemfiles/mysql2-0-5-0.gemfile + +env: + - SEMIAN_CIRCUIT_BREAKER_IMPL=worker + - SEMIAN_CIRCUIT_BREAKER_IMPL=host script: - - docker build --build-arg BUNDLE_GEMFILE="$GEMFILE" -t shopify/semian-ci:latest -f dockerfiles/semian-ci . + - | + docker build \ + --build-arg BUNDLE_GEMFILE="$BUNDLE_GEMFILE" \ + --build-arg SEMIAN_CIRCUIT_BREAKER_IMPL="$SEMIAN_CIRCUIT_BREAKER_IMPL" \ + -t shopify/semian-ci:latest -f dockerfiles/semian-ci . - travis_retry docker-compose -f docker-compose.ci.yml up --force-recreate --exit-code-from semian notifications: diff --git a/dockerfiles/semian-ci b/dockerfiles/semian-ci index 7af9afb6..2761ef6a 100644 --- a/dockerfiles/semian-ci +++ b/dockerfiles/semian-ci @@ -11,3 +11,6 @@ WORKDIR /app COPY . . ARG BUNDLE_GEMFILE ENV BUNDLE_GEMFILE="${BUNDLE_GEMFILE}" + +ARG SEMIAN_CIRCUIT_BREAKER_IMPL +ENV SEMIAN_CIRCUIT_BREAKER_IMPL="${SEMIAN_CIRCUIT_BREAKER_IMPL}" From 63495a9c02f976fcb5d1edc65a067e5d4d30632e Mon Sep 17 00:00:00 2001 From: Aditya Sharma Date: Tue, 25 Jun 2019 19:55:59 -0400 Subject: [PATCH 32/49] make workdir similar to travis path --- dockerfiles/semian-ci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/semian-ci b/dockerfiles/semian-ci index 2761ef6a..4313c2ff 100644 --- a/dockerfiles/semian-ci +++ b/dockerfiles/semian-ci @@ -7,7 +7,7 @@ RUN apt-get update \ RUN gem install bundler -WORKDIR /app +WORKDIR /home/travis/build/Shopify/semian COPY . . ARG BUNDLE_GEMFILE ENV BUNDLE_GEMFILE="${BUNDLE_GEMFILE}" From 13763b1c819df0ecc3e16a0cdbef3ba41955baf3 Mon Sep 17 00:00:00 2001 From: Aditya Sharma Date: Tue, 25 Jun 2019 20:17:42 -0400 Subject: [PATCH 33/49] Exclude running tests with host-based circuits for gemfiles --- .travis.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 733354bf..f84b8f31 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,12 +23,22 @@ env: - SEMIAN_CIRCUIT_BREAKER_IMPL=worker - SEMIAN_CIRCUIT_BREAKER_IMPL=host +matrix: + exclude: + - gemfile: gemfiles/hiredis-0-6.gemfile + env: SEMIAN_CIRCUIT_BREAKER_IMPL=host + - gemfile: gemfiles/mysql2-0-4-10.gemfile + env: SEMIAN_CIRCUIT_BREAKER_IMPL=host + - gemfile: gemfiles/Gemfile.rails-2.3.x + env: SEMIAN_CIRCUIT_BREAKER_IMPL=host + script: - | docker build \ --build-arg BUNDLE_GEMFILE="$BUNDLE_GEMFILE" \ --build-arg SEMIAN_CIRCUIT_BREAKER_IMPL="$SEMIAN_CIRCUIT_BREAKER_IMPL" \ - -t shopify/semian-ci:latest -f dockerfiles/semian-ci . + -t shopify/semian-ci:latest \ + -f dockerfiles/semian-ci . - travis_retry docker-compose -f docker-compose.ci.yml up --force-recreate --exit-code-from semian notifications: From e2f8a5d369b28999addc4718cbd2c8faa33bd079 Mon Sep 17 00:00:00 2001 From: Aditya Sharma Date: Tue, 25 Jun 2019 20:20:29 -0400 Subject: [PATCH 34/49] fix typo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f84b8f31..352b8fc0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ matrix: env: SEMIAN_CIRCUIT_BREAKER_IMPL=host - gemfile: gemfiles/mysql2-0-4-10.gemfile env: SEMIAN_CIRCUIT_BREAKER_IMPL=host - - gemfile: gemfiles/Gemfile.rails-2.3.x + - gemfile: gemfiles/mysql2-0-5-0.gemfile env: SEMIAN_CIRCUIT_BREAKER_IMPL=host script: From efd72d029d095bd9561bbbbe225beb0b4982def5 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Wed, 26 Jun 2019 13:21:08 -0400 Subject: [PATCH 35/49] Ignore BeyondCompare merge files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1bc40cbd..3834a68b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ /pkg *.gem *.log +*.orig /html/ Gemfile.lock vendor/ From d16c6ab3c35fdad477139f896061ec3c99b0dd1a Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Wed, 26 Jun 2019 17:12:18 -0400 Subject: [PATCH 36/49] Implement the scale_factor for host-based sliding windows --- ext/semian/resource.c | 2 + ext/semian/sliding_window.c | 78 +++++++++++++++++++--- ext/semian/sliding_window.h | 5 +- ext/semian/sysv_semaphores.c | 7 +- ext/semian/sysv_semaphores.h | 1 + ext/semian/types.h | 1 + lib/semian.rb | 1 + lib/semian/circuit_breaker.rb | 9 +-- lib/semian/simple_sliding_window.rb | 12 +++- test/adapter_test.rb | 2 +- test/circuit_breaker_test.rb | 5 +- test/grpc_test.rb | 4 +- test/helpers/circuit_breaker_helper.rb | 1 - test/lru_hash_test.rb | 4 +- test/net_http_test.rb | 2 +- test/resource_test.rb | 17 ++--- test/simple_sliding_window_test.rb | 90 +++++++++++++++++++------- test/test_helper.rb | 4 ++ 18 files changed, 184 insertions(+), 61 deletions(-) diff --git a/ext/semian/resource.c b/ext/semian/resource.c index 52144cfb..afd448df 100644 --- a/ext/semian/resource.c +++ b/ext/semian/resource.c @@ -337,6 +337,8 @@ static inline void semian_resource_free(void *ptr) { semian_resource_t *res = (semian_resource_t *) ptr; + dprintf("Freeing resource sem_id:%d", res->sem_id); + if (res->name) { free(res->name); res->name = NULL; diff --git a/ext/semian/sliding_window.c b/ext/semian/sliding_window.c index 6def16bb..6c6a8e0f 100644 --- a/ext/semian/sliding_window.c +++ b/ext/semian/sliding_window.c @@ -41,14 +41,18 @@ static int check_max_size_arg(VALUE max_size) { int retval = -1; - switch (TYPE(max_size)) { + switch (rb_type(max_size)) { case T_NIL: + case T_UNDEF: retval = SLIDING_WINDOW_MAX_SIZE; break; case T_FLOAT: rb_warn("semian sliding window max_size is a float, converting to fixnum"); retval = (int)(RFLOAT_VALUE(max_size)); break; - default: + case T_FIXNUM: + case T_BIGNUM: retval = RB_NUM2INT(max_size); break; + default: + rb_raise(rb_eArgError, "unknown type for max_size: %d", TYPE(max_size)); } if (retval <= 0) { @@ -60,6 +64,33 @@ check_max_size_arg(VALUE max_size) return retval; } +static float +check_scale_factor_arg(VALUE scale_factor) +{ + float retval = 1.0; + switch(rb_type(scale_factor)) { + case T_NIL: + case T_UNDEF: + retval = 1.0; break; + case T_FLOAT: + retval = rb_float_value(scale_factor); break; + case T_FIXNUM: + case T_BIGNUM: + rb_warn("semian sliding window scale_factor is an int, converting to float"); + retval = (float)RB_NUM2INT(scale_factor); break; + default: + rb_raise(rb_eArgError, "unknown type for scale_factor: %d", TYPE(scale_factor)); + } + + if (retval <= 0.0) { + rb_raise(rb_eArgError, "scale_factor must be greater than zero"); + } else if (retval > 1.0) { + rb_raise(rb_eArgError, "scale_factor cannot be greater than 1.0"); + } + + return retval; +} + static VALUE grow_window(semian_simple_sliding_window_shared_t* window, int new_max_size) { @@ -106,12 +137,12 @@ shrink_window(semian_simple_sliding_window_shared_t* window, int new_max_size) window->end = 0; } else if (window->end > window->start) { // Easy case - the window doesn't wrap around - window->end = window->start + new_length; + window->start = window->start + new_length; } else { // Hard case - the window wraps, so re-index the data // Adapted from http://www.cplusplus.com/reference/algorithm/rotate/ int first = 0; - int middle = window->start; + int middle = (window->end - new_max_size + window->max_size) % window->max_size; int last = window->max_size; int next = middle; while (first != next) { @@ -169,10 +200,11 @@ Init_SlidingWindow() VALUE cSlidingWindow = rb_const_get(cSimple, rb_intern("SlidingWindow")); rb_define_alloc_func(cSlidingWindow, semian_simple_sliding_window_alloc); - rb_define_method(cSlidingWindow, "initialize_sliding_window", semian_simple_sliding_window_initialize, 2); + rb_define_method(cSlidingWindow, "initialize_sliding_window", semian_simple_sliding_window_initialize, 3); rb_define_method(cSlidingWindow, "size", semian_simple_sliding_window_size, 0); rb_define_method(cSlidingWindow, "resize_to", semian_simple_sliding_window_resize_to, 1); - rb_define_method(cSlidingWindow, "max_size", semian_simple_sliding_window_max_size, 0); + rb_define_method(cSlidingWindow, "max_size", semian_simple_sliding_window_max_size_get, 0); + rb_define_method(cSlidingWindow, "max_size=", semian_simple_sliding_window_max_size_set, 1); rb_define_method(cSlidingWindow, "values", semian_simple_sliding_window_values, 0); rb_define_method(cSlidingWindow, "last", semian_simple_sliding_window_last, 0); rb_define_method(cSlidingWindow, "<<", semian_simple_sliding_window_push, 1); @@ -212,7 +244,7 @@ static int max(int a, int b) { } VALUE -semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_size) +semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_size, VALUE scale_factor) { semian_simple_sliding_window_t *res = get_object(self); @@ -228,13 +260,15 @@ semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_size) res->sem_id = initialize_single_semaphore(res->key, SEM_DEFAULT_PERMISSIONS); res->shmem = get_or_create_shared_memory(res->key, init_fn); res->error_threshold = check_max_size_arg(max_size); + res->scale_factor = check_scale_factor_arg(scale_factor); sem_meta_lock(res->sem_id); { int workers = get_number_of_registered_workers(res); - float scale_factor = (workers > 1) ? 0.2 : 1.0; // TODO: Parameterize - int error_threshold = max(res->error_threshold, (int) ceil(workers * scale_factor * res->error_threshold)); + float scale = (workers > 1) ? res->scale_factor : 1.0; // TODO: Parameterize + int error_threshold = max(res->error_threshold, (int) ceil(workers * scale * res->error_threshold)); + dprintf(" workers:%d scale:%0.2f error_threshold:%d", workers, scale, error_threshold); resize_window(res->shmem, error_threshold); } sem_meta_unlock(res->sem_id); @@ -264,6 +298,10 @@ semian_simple_sliding_window_resize_to(VALUE self, VALUE new_size) VALUE retval = Qnil; int new_max_size = RB_NUM2INT(new_size); + if (new_max_size < 1) { + rb_raise(rb_eArgError, "cannot resize to %d", new_max_size); + } + sem_meta_lock(res->sem_id); { retval = resize_window(res->shmem, new_max_size); @@ -274,7 +312,7 @@ semian_simple_sliding_window_resize_to(VALUE self, VALUE new_size) } VALUE -semian_simple_sliding_window_max_size(VALUE self) +semian_simple_sliding_window_max_size_get(VALUE self) { semian_simple_sliding_window_t *res = get_object(self); VALUE retval; @@ -288,6 +326,26 @@ semian_simple_sliding_window_max_size(VALUE self) return retval; } +VALUE +semian_simple_sliding_window_max_size_set(VALUE self, VALUE new_size) +{ + semian_simple_sliding_window_t *res = get_object(self); + VALUE retval; + + int new_max_size = RB_NUM2INT(new_size); + if (new_max_size < 1) { + rb_raise(rb_eArgError, "max_size must be positive"); + } + + sem_meta_lock(res->sem_id); + { + retval = resize_window(res->shmem, new_max_size); + } + sem_meta_unlock(res->sem_id); + + return retval; +} + VALUE semian_simple_sliding_window_values(VALUE self) { diff --git a/ext/semian/sliding_window.h b/ext/semian/sliding_window.h index 41f3ceba..c9471a06 100644 --- a/ext/semian/sliding_window.h +++ b/ext/semian/sliding_window.h @@ -7,10 +7,11 @@ void Init_SlidingWindow(); VALUE semian_simple_sliding_window_alloc(VALUE klass); -VALUE semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_size); +VALUE semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_size, VALUE scale_factor); VALUE semian_simple_sliding_window_size(VALUE self); VALUE semian_simple_sliding_window_resize_to(VALUE self, VALUE new_size); -VALUE semian_simple_sliding_window_max_size(VALUE self); +VALUE semian_simple_sliding_window_max_size_get(VALUE self); +VALUE semian_simple_sliding_window_max_size_set(VALUE self, VALUE new_size); VALUE semian_simple_sliding_window_push(VALUE self, VALUE value); VALUE semian_simple_sliding_window_values(VALUE self); VALUE semian_simple_sliding_window_last(VALUE self); diff --git a/ext/semian/sysv_semaphores.c b/ext/semian/sysv_semaphores.c index 367af3d3..22ede193 100644 --- a/ext/semian/sysv_semaphores.c +++ b/ext/semian/sysv_semaphores.c @@ -29,8 +29,9 @@ raise_semian_syscall_error(const char *syscall, int error_num) void initialize_semaphore_set(semian_resource_t* res, const char* id_str, long permissions, int tickets, double quota) { - res->key = generate_key(id_str); + dprintf("Initializing semaphore set for key:%lu", res->key); + res->strkey = (char*) malloc((2 /*for 0x*/+ sizeof(uint64_t) /*actual key*/+ 1 /*null*/) * sizeof(char)); sprintf(res->strkey, "0x%08x", (unsigned int) res->key); @@ -60,6 +61,7 @@ initialize_semaphore_set(semian_resource_t* res, const char* id_str, long permis Ensure that a worker for this process is registered. Note that from ruby we ensure that at most one worker may be registered per process. */ + dprintf("Registering worker for sem_id:%d", res->sem_id); if (perform_semop(res->sem_id, SI_SEM_REGISTERED_WORKERS, 1, SEM_UNDO, NULL) == -1) { rb_raise(eInternal, "error incrementing registered workers, errno: %d (%s)", errno, strerror(errno)); } @@ -120,10 +122,12 @@ perform_semop(int sem_id, short index, short op, short flags, struct timespec *t int get_sem_val(int sem_id, int sem_index) { + dprintf("get_sem_val(sem_id: %d, sem_index: %d)", sem_id, sem_index); int ret = semctl(sem_id, sem_index, GETVAL); if (ret == -1) { rb_raise(eInternal, "error getting value of %s for sem %d, errno: %d (%s)", SEMINDEX_STRING[sem_index], sem_id, errno, strerror(errno)); } + dprintf("get_sem_val(sem_id: %d, sem_index: %d) == %d", sem_id, sem_index, ret); return ret; } @@ -245,6 +249,7 @@ diff_timespec_ms(struct timespec *end, struct timespec *begin) int initialize_single_semaphore(uint64_t key, long permissions) { + dprintf("Initializing single semaphore for key:%lu", key); int sem_id = semget(key, 1, IPC_CREAT | IPC_EXCL | permissions); /* diff --git a/ext/semian/sysv_semaphores.h b/ext/semian/sysv_semaphores.h index d680fd0f..e170df5e 100644 --- a/ext/semian/sysv_semaphores.h +++ b/ext/semian/sysv_semaphores.h @@ -118,6 +118,7 @@ initialize_single_semaphore(uint64_t key, long permissions); static inline void dprint_sem_vals(int sem_id) { + dprintf("dprintf(%d)", sem_id); dprintf("sem_id: %d, lock: %d, tickets: %d configured: %d, registered workers %d", sem_id, get_sem_val(sem_id, SI_SEM_LOCK), diff --git a/ext/semian/types.h b/ext/semian/types.h index 4bca785a..7935ac5e 100644 --- a/ext/semian/types.h +++ b/ext/semian/types.h @@ -78,6 +78,7 @@ typedef struct { int sem_id; uint64_t parent_key; int error_threshold; + float scale_factor; semian_simple_sliding_window_shared_t* shmem; } semian_simple_sliding_window_t; diff --git a/lib/semian.rb b/lib/semian.rb index 7cdd504f..d579f774 100644 --- a/lib/semian.rb +++ b/lib/semian.rb @@ -259,6 +259,7 @@ def create_circuit_breaker(name, **options) exceptions: Array(exceptions) + [::Semian::BaseError], half_open_resource_timeout: options[:half_open_resource_timeout], implementation: implementation(**options), + scale_factor: options[:scale_factor], ) end diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index c5ca5105..8791338d 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -7,17 +7,18 @@ class CircuitBreaker #:nodoc: attr_reader :name, :half_open_resource_timeout, :error_timeout, :state, :last_error def initialize(name, exceptions:, success_threshold:, error_threshold:, - error_timeout:, implementation:, half_open_resource_timeout: nil) - @name = name.to_sym - initialize_circuit_breaker(name, error_threshold) + error_timeout:, implementation:, half_open_resource_timeout: nil, scale_factor: nil) + initialize_circuit_breaker(name, error_threshold) if respond_to?(:initialize_circuit_breaker) + @name = name.to_sym @success_count_threshold = success_threshold @error_count_threshold = error_threshold + @scale_factor = scale_factor @error_timeout = error_timeout @exceptions = exceptions @half_open_resource_timeout = half_open_resource_timeout - @errors = implementation::SlidingWindow.new(name, max_size: @error_count_threshold) + @errors = implementation::SlidingWindow.new(name, max_size: @error_count_threshold, scale_factor: @scale_factor) @successes = implementation::Integer.new("#{name}_successes") state_val = implementation::Integer.new("#{name}_state") @state = implementation::State.new(state_val) diff --git a/lib/semian/simple_sliding_window.rb b/lib/semian/simple_sliding_window.rb index bd0c2ed8..3befd0c1 100644 --- a/lib/semian/simple_sliding_window.rb +++ b/lib/semian/simple_sliding_window.rb @@ -9,8 +9,10 @@ class SlidingWindow #:nodoc: # like this: if @max_size = 4, current time is 10, @window =[5,7,9,10]. # Another push of (11) at 11 sec would make @window [7,9,10,11], shifting off 5. - def initialize(name, max_size:) - initialize_sliding_window(name, max_size) + def initialize(name, max_size:, scale_factor: nil) + initialize_sliding_window(name, max_size, scale_factor) if respond_to?(:initialize_sliding_window) + + @name = name.to_sym @max_size = max_size @window = [] end @@ -44,6 +46,12 @@ def clear end alias_method :destroy, :clear + def max_size=(value) + raise ArgumentError, "max_size must be positive" if value <= 0 + @max_size = value + resize_to(value) + end + private def resize_to(size) diff --git a/test/adapter_test.rb b/test/adapter_test.rb index 01c561f0..55a9549d 100644 --- a/test/adapter_test.rb +++ b/test/adapter_test.rb @@ -21,7 +21,7 @@ def test_adapter_registers_consumer end def test_unregister - skip if ENV["SKIP_FLAKY_TESTS"] + skip if flaky client = Semian::AdapterTestClient.new(quota: 0.5) assert_nil(Semian.resources[:testing]) resource = Semian.register(:testing, tickets: 2, error_threshold: 1, error_timeout: 0, success_threshold: 0) diff --git a/test/circuit_breaker_test.rb b/test/circuit_breaker_test.rb index eacd1776..8d4275bf 100644 --- a/test/circuit_breaker_test.rb +++ b/test/circuit_breaker_test.rb @@ -25,7 +25,6 @@ def test_acquire_yield_when_the_circuit_is_closed def test_acquire_raises_circuit_open_error_when_the_circuit_is_open open_circuit! assert_raises Semian::OpenCircuitError do - puts "test: Acquiring resource #{@resource.circuit_breaker}" @resource.acquire { 1 + 1 } end assert_match(/State transition from closed to open/, @strio.string) @@ -150,6 +149,10 @@ def test_semian_wide_env_var_disables_circuit_breaker end class RawResource + def initialize + @timeout = 2 + end + def timeout @timeout || 2 end diff --git a/test/grpc_test.rb b/test/grpc_test.rb index 6fc8a630..0ecb1095 100644 --- a/test/grpc_test.rb +++ b/test/grpc_test.rb @@ -64,7 +64,7 @@ def test_unavailable_server_opens_the_circuit end def test_timeout_opens_the_circuit - skip if ENV["SKIP_FLAKY_TESTS"] + skip if flaky stub = build_insecure_stub(EchoStub, host: "#{SemianConfig['toxiproxy_upstream_host']}:#{SemianConfig['grpc_toxiproxy_port']}", opts: {timeout: 0.1}) run_services_on_server(@server, services: [EchoService]) do Toxiproxy['semian_test_grpc'].downstream(:latency, latency: 1000).apply do @@ -86,7 +86,7 @@ def test_timeout_opens_the_circuit def test_instrumentation notified = false subscriber = Semian.subscribe do |event, resource, scope, adapter| - next if event != :success + next unless event == :success notified = true assert_equal Semian[@host], resource diff --git a/test/helpers/circuit_breaker_helper.rb b/test/helpers/circuit_breaker_helper.rb index 764effa7..55fda301 100644 --- a/test/helpers/circuit_breaker_helper.rb +++ b/test/helpers/circuit_breaker_helper.rb @@ -15,7 +15,6 @@ def half_open_cicuit!(resource = @resource, backwards_time_travel = 10) def trigger_error!(resource = @resource, error = SomeError) resource.acquire do - puts "Triggering error" raise error end rescue error diff --git a/test/lru_hash_test.rb b/test/lru_hash_test.rb index 0793f30a..138a97ca 100644 --- a/test/lru_hash_test.rb +++ b/test/lru_hash_test.rb @@ -113,8 +113,9 @@ def test_clean_instrumentation notified = false subscriber = Semian.subscribe do |event, resource, scope, adapter, payload| + next unless event == :lru_hash_gc + notified = true - assert_equal :lru_hash_gc, event assert_equal @lru_hash, resource assert_nil scope assert_nil adapter @@ -234,6 +235,7 @@ def create_circuit_breaker(name, exceptions = true, bulkhead = false, error_time exceptions: [::Semian::BaseError], half_open_resource_timeout: nil, implementation: implementation, + scale_factor: 1.0, ) circuit_breaker.mark_failed(nil) if exceptions Semian::ProtectedResource.new(name, create_bulkhead(name, bulkhead), circuit_breaker) diff --git a/test/net_http_test.rb b/test/net_http_test.rb index 726d9d36..33672034 100644 --- a/test/net_http_test.rb +++ b/test/net_http_test.rb @@ -361,7 +361,7 @@ def test_5xxs_trip_circuit_when_fatal_server_flag_enabled end def test_5xxs_dont_raise_exceptions_unless_fatal_server_flag_enabled - skip if ENV["SKIP_FLAKY_TESTS"] + skip if flaky with_semian_configuration do with_server do http = Net::HTTP.new(SemianConfig['http_host'], SemianConfig['http_port_service_a']) diff --git a/test/resource_test.rb b/test/resource_test.rb index ca4a3b12..79dd7885 100644 --- a/test/resource_test.rb +++ b/test/resource_test.rb @@ -4,11 +4,9 @@ class TestResource < Minitest::Test include ResourceHelper - # Time epsilon to account for super fast machines - EPSILON = 0.1 - def setup @workers = [] + @resources = [] Semian.destroy(:testing) rescue nil @@ -143,10 +141,9 @@ def test_acquire_return_val end def test_acquire_timeout - fork_workers(count: 2, tickets: 1, timeout: 1, wait_for_timeout: true) + fork_workers(count: 2, tickets: 1, timeout: 1, wait_for_timeout: true, epsilon: 0.1) signal_workers('TERM') - timeouts = count_worker_timeouts - assert 1, timeouts + assert_equal(1, count_worker_timeouts) end def test_acquire_timeout_override @@ -158,8 +155,7 @@ def test_acquire_timeout_override signal_workers('TERM') - timeouts = count_worker_timeouts - assert 0, timeouts + assert_equal(0, count_worker_timeouts) end def test_acquire_with_fork @@ -522,7 +518,7 @@ def destroy_resources # Active workers are accumulated in the instance variable @workers, # and workers must be cleaned up between tests by the teardown script # An exit value of 100 is to keep track of timeouts, 0 for success. - def fork_workers(count:, resource: :testing, quota: nil, tickets: nil, timeout: 0.1, wait_for_timeout: false) + def fork_workers(count:, resource: :testing, quota: nil, tickets: nil, timeout: 0.1, wait_for_timeout: false, epsilon: 0) fail 'Must provide at least one of tickets or quota' unless tickets || quota @workers ||= [] @@ -548,13 +544,12 @@ def fork_workers(count:, resource: :testing, quota: nil, tickets: nil, timeout: end sleep rescue => e - puts "Unhandled exception occurred in worker" puts e exit! 2 end end end - sleep((count / 2.0).ceil * timeout + EPSILON) if wait_for_timeout # give time for threads to timeout + sleep((count / 2.0).ceil * timeout + epsilon) if wait_for_timeout # give time for threads to timeout end def count_worker_timeouts diff --git a/test/simple_sliding_window_test.rb b/test/simple_sliding_window_test.rb index 35c55a1d..24de1d31 100644 --- a/test/simple_sliding_window_test.rb +++ b/test/simple_sliding_window_test.rb @@ -7,10 +7,6 @@ def setup @sliding_window.clear end - def teardown - @sliding_window.destroy - end - def test_clear id = Time.now.strftime('%H:%M:%S.%N') window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 6) @@ -52,7 +48,8 @@ def test_sliding_window_reject end def test_sliding_window_reject_failure - skip if ENV['SEMIAN_CIRCUIT_BREAKER_IMPL'] == 'worker' + skip unless ENV['SEMIAN_CIRCUIT_BREAKER_IMPL'] == 'host' + @sliding_window << 0 << 1 << 2 << 3 << 4 << 5 << 6 << 7 assert_equal(6, @sliding_window.size) assert_sliding_window(@sliding_window, [2, 3, 4, 5, 6, 7], 6) @@ -64,13 +61,14 @@ def test_sliding_window_reject_failure def test_resize_to_less_than_1_raises assert_raises ArgumentError do - @sliding_window.resize_to 0 + @sliding_window.max_size = 0 end end def test_resize_to_1_works - assert_equal(0, @sliding_window.size) - @sliding_window.resize_to 1 + assert_sliding_window(@sliding_window, [], 6) + @sliding_window.max_size = 1 + assert_sliding_window(@sliding_window, [], 1) end def test_resize_to_simple @@ -78,7 +76,7 @@ def test_resize_to_simple window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) window << 0 << 1 assert_sliding_window(window, [0, 1], 4) - window.resize_to(8) + window.max_size = 8 assert_sliding_window(window, [0, 1], 8) window << 2 << 3 << 4 << 5 assert_sliding_window(window, [0, 1, 2, 3, 4, 5], 8) @@ -89,7 +87,7 @@ def test_resize_to_simple_full window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) window << 0 << 1 << 2 << 3 assert_sliding_window(window, [0, 1, 2, 3], 4) - window.resize_to(8) + window.max_size = 8 assert_sliding_window(window, [0, 1, 2, 3], 8) window << 4 << 5 assert_sliding_window(window, [0, 1, 2, 3, 4, 5], 8) @@ -102,7 +100,7 @@ def test_resize_to_simple_floating assert_sliding_window(window, [0, 1, 2, 3], 4) window.reject! { |val| val < 2 } assert_sliding_window(window, [2, 3], 4) - window.resize_to(8) + window.max_size = 8 assert_sliding_window(window, [2, 3], 8) window << 4 << 5 assert_sliding_window(window, [2, 3, 4, 5], 8) @@ -113,7 +111,7 @@ def test_resize_to_hard window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) window << 0 << 1 << 2 << 3 << 4 << 5 assert_sliding_window(window, [2, 3, 4, 5], 4) - window.resize_to(8) + window.max_size = 8 assert_sliding_window(window, [2, 3, 4, 5], 8) window << 6 << 7 assert_sliding_window(window, [2, 3, 4, 5, 6, 7], 8) @@ -126,7 +124,7 @@ def test_resize_to_shrink_simple window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) window << 0 << 1 assert_sliding_window(window, [0, 1], 4) - window.resize_to(2) + window.max_size = 2 assert_sliding_window(window, [0, 1], 2) end @@ -135,8 +133,8 @@ def test_resize_to_shrink_simple_full window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) window << 0 << 1 << 2 << 3 assert_sliding_window(window, [0, 1, 2, 3], 4) - window.resize_to(2) - assert_sliding_window(window, [0, 1], 2) + window.max_size = 2 + assert_sliding_window(window, [2, 3], 2) end def test_resize_to_shrink_hard @@ -144,30 +142,30 @@ def test_resize_to_shrink_hard window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) window << 0 << 1 << 2 << 3 << 4 << 5 assert_sliding_window(window, [2, 3, 4, 5], 4) - window.resize_to(2) - assert_sliding_window(window, [2, 3], 2) + window.max_size = 2 + assert_sliding_window(window, [4, 5], 2) end def test_resize_to_shrink_all_index - 4.times do |offset| - id = Time.now.strftime('%H:%M:%S.%N-#{offset}') + 8.times do |offset| + id = Time.now.strftime("%H:%M:%S.%N-#{offset}") window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) offset.times { window << offset } window << 0 << 1 << 2 << 3 assert_sliding_window(window, [0, 1, 2, 3], 4) - window.resize_to(2) - assert_sliding_window(window, [0, 1], 2) + window.max_size = 2 + assert_sliding_window(window, [2, 3], 2) end end def test_resize_to_grow_all_index - 4.times do |offset| - id = Time.now.strftime('%H:%M:%S.%N-#{offset}') + 8.times do |offset| + id = Time.now.strftime("%H:%M:%S.%N-#{offset}") window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 4) offset.times { window << offset } window << 0 << 1 << 2 << 3 assert_sliding_window(window, [0, 1, 2, 3], 4) - window.resize_to(8) + window.max_size = 8 assert_sliding_window(window, [0, 1, 2, 3], 8) window << 4 << 5 << 6 << 7 assert_sliding_window(window, [0, 1, 2, 3, 4, 5, 6, 7], 8) @@ -176,6 +174,50 @@ def test_resize_to_grow_all_index end end + def test_scale_factor + pids = [] + skip unless ENV['SEMIAN_CIRCUIT_BREAKER_IMPL'] == 'host' + + id = Time.now.strftime("%H:%M:%S.%N") + window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 1, scale_factor: 0.5) + + fork_worker = Proc.new do |i| + pid = fork do + # TODO(michaelkipper): We need to create a bulkhead here, because the sliding window + # has a coupling to it via the registered worker semaphore. This + # sucks, and should be removed by refactoring the bulkhead out of + # the Semian resource concept. + ::Semian::Resource.new(id, tickets: 1) + window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 10, scale_factor: 0.5) + window << (i * 100) + sleep(3600) + end + + puts "Forker worker #{pid}" + sleep(0.1) + pid + end + + pids << fork_worker.call(1) + assert_equal(10, window.max_size) + pids << fork_worker.call(2) + assert_equal(10, window.max_size) + pids << fork_worker.call(3) + assert_equal(15, window.max_size) + pids << fork_worker.call(4) + assert_equal(20, window.max_size) + + assert_equal(4, window.size) + ensure + pids.each do |pid| + begin + Process.kill('TERM', pid) + rescue + nil + end + end + end + private def assert_sliding_window(sliding_window, array, max_size) diff --git a/test/test_helper.rb b/test/test_helper.rb index 32f0d63e..ff93af5f 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -62,3 +62,7 @@ class Minitest::Test include BackgroundHelper end + +def flaky + ENV["SKIP_FLAKY_TESTS"] +end From ddd7268dccf3ee2ae192c0a2616d972fadd3c1e1 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Sun, 30 Jun 2019 09:51:25 -0400 Subject: [PATCH 37/49] Enabled debug messages with SEMIAN_DEBUG --- ext/semian/extconf.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/semian/extconf.rb b/ext/semian/extconf.rb index ff000666..88ab4e3f 100644 --- a/ext/semian/extconf.rb +++ b/ext/semian/extconf.rb @@ -24,7 +24,7 @@ have_func 'rb_thread_call_without_gvl' $CFLAGS = "-D_GNU_SOURCE -Werror -Wall -std=gnu99 " -if ENV.key?('DEBUG') +if ENV.key?('DEBUG') || ENV.key?('SEMIAN_DEBUG') $CFLAGS << "-O0 -g -DDEBUG" else $CFLAGS << "-O3" From 0a00e9434be21b4213fbc144b6e288b98379b7aa Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Sun, 30 Jun 2019 17:20:22 -0400 Subject: [PATCH 38/49] Store SimpleInteger in semaphores, not shared memory --- Gemfile | 1 + Rakefile | 4 +- ext/semian/circuit_breaker.c | 67 --------------------- ext/semian/circuit_breaker.h | 12 ---- ext/semian/semian.c | 9 +-- ext/semian/semian.h | 1 - ext/semian/simple_integer.c | 92 ++++++++++++++--------------- ext/semian/sliding_window.c | 7 ++- ext/semian/sysv_semaphores.c | 50 ++++++++-------- ext/semian/sysv_semaphores.h | 14 +++-- ext/semian/tickets.c | 2 +- ext/semian/types.h | 17 ------ lib/semian/circuit_breaker.rb | 2 - lib/semian/simple_integer.rb | 6 ++ lib/semian/simple_sliding_window.rb | 19 +++++- test/simple_integer_test.rb | 40 +++++++++---- test/simple_sliding_window_test.rb | 10 ++++ test/test_helper.rb | 3 + 18 files changed, 158 insertions(+), 198 deletions(-) delete mode 100644 ext/semian/circuit_breaker.c delete mode 100644 ext/semian/circuit_breaker.h diff --git a/Gemfile b/Gemfile index 383b1c51..f4ef742b 100644 --- a/Gemfile +++ b/Gemfile @@ -3,4 +3,5 @@ gemspec group :development, :test do gem 'rubocop' + gem 'minitest-reporters' end diff --git a/Rakefile b/Rakefile index b111f0e9..26cce7ee 100644 --- a/Rakefile +++ b/Rakefile @@ -41,8 +41,8 @@ require 'rake/testtask' Rake::TestTask.new 'test' do |t| t.libs = %w(lib test) t.pattern = "test/*_test.rb" - t.verbose = false - t.warning = false + t.verbose = true + t.warning = true end # ========================================================== diff --git a/ext/semian/circuit_breaker.c b/ext/semian/circuit_breaker.c deleted file mode 100644 index f36034a6..00000000 --- a/ext/semian/circuit_breaker.c +++ /dev/null @@ -1,67 +0,0 @@ -#include "circuit_breaker.h" - -#include "sysv_semaphores.h" -#include "sysv_shared_memory.h" -#include "types.h" -#include "util.h" - -void -semian_circuit_breaker_free(void* ptr) -{ - semian_circuit_breaker_t* res = (semian_circuit_breaker_t*)ptr; - free_shared_memory(res->shmem); -} - -size_t -semian_circuit_breaker_size(const void* ptr) -{ - return sizeof(semian_circuit_breaker_t); -} - -static const rb_data_type_t semian_circuit_breaker_type = { - .wrap_struct_name = "semian_circuit_breaker", - .function = { - .dmark = NULL, - .dfree = semian_circuit_breaker_free, - .dsize = semian_circuit_breaker_size, - }, - .data = NULL, - .flags = RUBY_TYPED_FREE_IMMEDIATELY, -}; - -void -Init_CircuitBreaker() { - dprintf("Init_CircuitBreaker"); - - VALUE cSemian = rb_const_get(rb_cObject, rb_intern("Semian")); - VALUE cCircuitBreaker = rb_const_get(cSemian, rb_intern("CircuitBreaker")); - - rb_define_alloc_func(cCircuitBreaker, semian_circuit_breaker_alloc); - rb_define_method(cCircuitBreaker, "initialize_circuit_breaker", semian_circuit_breaker_initialize, 2); -} - -VALUE -semian_circuit_breaker_alloc(VALUE klass) -{ - semian_circuit_breaker_t *res; - VALUE obj = TypedData_Make_Struct(klass, semian_circuit_breaker_t, &semian_circuit_breaker_type, res); - return obj; -} - -VALUE -semian_circuit_breaker_initialize(VALUE self, VALUE name) -{ - semian_circuit_breaker_t *res = NULL; - TypedData_Get_Struct(self, semian_circuit_breaker_t, &semian_circuit_breaker_type, res); - - char buffer[1024]; - strcpy(buffer, to_s(name)); - strcat(buffer, "_circuit_breaker"); - res->key = generate_key(buffer); - - dprintf("Initializing circuit breaker '%s' (key: %lu)", buffer, res->key); - res->sem_id = initialize_single_semaphore(res->key, SEM_DEFAULT_PERMISSIONS); - res->shmem = get_or_create_shared_memory(res->key, NULL); - - return self; -} diff --git a/ext/semian/circuit_breaker.h b/ext/semian/circuit_breaker.h deleted file mode 100644 index 0bbb5923..00000000 --- a/ext/semian/circuit_breaker.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef CIRCUIT_BREAKER_H -#define CIRCUIT_BREAKER_H - -#include - -void Init_CircuitBreaker(); - -VALUE semian_circuit_breaker_alloc(VALUE klass); -VALUE semian_circuit_breaker_initialize(VALUE self, VALUE id); -VALUE semian_circuit_breaker_successes(VALUE self); - -#endif // CIRCUIT_BREAKER_H \ No newline at end of file diff --git a/ext/semian/semian.c b/ext/semian/semian.c index 34a0271c..cbabfb7b 100644 --- a/ext/semian/semian.c +++ b/ext/semian/semian.c @@ -70,7 +70,6 @@ void Init_semian() if (use_c_circuits()) { Init_SimpleInteger(); Init_SlidingWindow(); - Init_CircuitBreaker(); } } @@ -78,18 +77,20 @@ static int use_c_circuits() { char *circuit_impl = getenv("SEMIAN_CIRCUIT_BREAKER_IMPL"); if (circuit_impl == NULL) { - fprintf(stderr, "Warning: Defaulting to worker-based circuit breaker implementation\n"); + fprintf(stderr, "Warning: Defaulting to Semian worker-based circuit breaker implementation\n"); return 0; } else { if (!strcmp(circuit_impl, "worker")) { + fprintf(stderr, "Info: Semian using worker-based circuit implementation\n"); return 0; } else if (!strcmp(circuit_impl, "host")) { + fprintf(stderr, "Info: Semian using host-based circuit implementation\n"); return 1; } else { - fprintf(stderr, "Warning: Unknown circuit breaker implementation: '%s'\n", circuit_impl); + fprintf(stderr, "Warning: Unknown Semian circuit breaker implementation: '%s'\n", circuit_impl); return 0; } } - rb_raise(rb_eArgError, "Unknown circuit breaker implementation"); + rb_raise(rb_eArgError, "Unknown Semian circuit breaker implementation"); } diff --git a/ext/semian/semian.h b/ext/semian/semian.h index c38636e0..000bd593 100644 --- a/ext/semian/semian.h +++ b/ext/semian/semian.h @@ -7,7 +7,6 @@ Implements Init_semian, which is used as C/Ruby entrypoint. #ifndef SEMIAN_H #define SEMIAN_H -#include "circuit_breaker.h" #include "resource.h" #include "simple_integer.h" #include "sliding_window.h" diff --git a/ext/semian/simple_integer.c b/ext/semian/simple_integer.c index 7ba28398..84a68b9c 100644 --- a/ext/semian/simple_integer.c +++ b/ext/semian/simple_integer.c @@ -1,15 +1,12 @@ #include "simple_integer.h" #include "sysv_semaphores.h" -#include "sysv_shared_memory.h" #include "types.h" #include "util.h" void semian_simple_integer_dfree(void* ptr) { - semian_simple_integer_t* res = (semian_simple_integer_t*)ptr; - free_shared_memory(res->shmem); } size_t @@ -29,10 +26,25 @@ static const rb_data_type_t semian_simple_integer_type = { .flags = RUBY_TYPED_FREE_IMMEDIATELY, }; -static void init_fn(void* ptr) +int check_increment_arg(VALUE val) { - semian_simple_integer_shared_t* res = (semian_simple_integer_shared_t*)ptr; - res->val = 0; + VALUE retval; + + switch (rb_type(val)) { + case T_NIL: + case T_UNDEF: + retval = 1; break; + case T_FLOAT: + rb_warn("incrementing SingleInteger by a floating point value, converting to fixnum"); + retval = (int)(RFLOAT_VALUE(val)); break; + case T_FIXNUM: + case T_BIGNUM: + retval = RB_NUM2INT(val); break; + default: + rb_raise(rb_eArgError, "unknown type for val: %d", TYPE(val)); + } + + return retval; } void @@ -68,8 +80,7 @@ semian_simple_integer_initialize(VALUE self, VALUE name) res->key = generate_key(to_s(name)); dprintf("Initializing simple integer '%s' (key: %lu)", to_s(name), res->key); - res->sem_id = initialize_single_semaphore(res->key, SEM_DEFAULT_PERMISSIONS); - res->shmem = get_or_create_shared_memory(res->key, &init_fn); + res->sem_id = initialize_single_semaphore(res->key, SEM_DEFAULT_PERMISSIONS, 0); return self; } @@ -77,81 +88,68 @@ semian_simple_integer_initialize(VALUE self, VALUE name) VALUE semian_simple_integer_increment(int argc, VALUE *argv, VALUE self) { + semian_simple_integer_t *res; + TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); + // This is definitely the worst API ever. // https://silverhammermba.github.io/emberb/c/#parsing-arguments VALUE val; - semian_simple_integer_t *res; - VALUE retval; - rb_scan_args(argc, argv, "01", &val); - TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); + int value = check_increment_arg(val); + if (perform_semop(res->sem_id, 0, value, SEM_UNDO, NULL) == -1) { + rb_raise(eInternal, "error incrementing simple integer, errno: %d (%s)", errno, strerror(errno)); + } - sem_meta_lock(res->sem_id); - { - if (NIL_P(val)) { - res->shmem->val += 1; - } else { - res->shmem->val += RB_NUM2INT(val); - } - retval = RB_INT2NUM(res->shmem->val); + // Return the current value, but know that there is a race condition here: + // It's not necessarily the same value after incrementing above, since + // semop() doesn't return the modified value. + int retval = get_sem_val(res->sem_id, 0); + if (retval == -1) { + rb_raise(eInternal, "error getting simple integer, errno: %d (%s)", errno, strerror(errno)); } - sem_meta_unlock(res->sem_id); - return retval; + return RB_INT2NUM(retval); } VALUE semian_simple_integer_reset(VALUE self) { semian_simple_integer_t *res; - VALUE retval; - TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); - sem_meta_lock(res->sem_id); - { - res->shmem->val = 0; - retval = RB_INT2NUM(res->shmem->val); + if (set_sem_val(res->sem_id, 0, 0) == -1) { + rb_raise(eInternal, "error resetting simple integer, errno: %d (%s)", errno, strerror(errno)); } - sem_meta_unlock(res->sem_id); - return retval; + return Qnil; } VALUE semian_simple_integer_value_get(VALUE self) { semian_simple_integer_t *res; - VALUE retval; - TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); - sem_meta_lock(res->sem_id); - { - retval = RB_INT2NUM(res->shmem->val); + int val = get_sem_val(res->sem_id, 0); + if (val == -1) { + rb_raise(eInternal, "error getting simple integer, errno: %d (%s)", errno, strerror(errno)); } - sem_meta_unlock(res->sem_id); - return retval; + return RB_INT2NUM(val); } VALUE semian_simple_integer_value_set(VALUE self, VALUE val) { semian_simple_integer_t *res; - VALUE retval; - TypedData_Get_Struct(self, semian_simple_integer_t, &semian_simple_integer_type, res); - sem_meta_lock(res->sem_id); - { - // TODO(michaelkipper): Check for respond_to?(:to_i) before calling. - VALUE to_i = rb_funcall(val, rb_intern("to_i"), 0); - res->shmem->val = RB_NUM2INT(to_i); - retval = RB_INT2NUM(res->shmem->val); + VALUE to_i = rb_funcall(val, rb_intern("to_i"), 0); + int value = RB_NUM2INT(to_i); + if (set_sem_val(res->sem_id, 0, value) == -1) { + rb_raise(eInternal, "error setting simple integer, errno: %d (%s)", errno, strerror(errno)); } - sem_meta_unlock(res->sem_id); - return retval; + return Qnil; } diff --git a/ext/semian/sliding_window.c b/ext/semian/sliding_window.c index 6c6a8e0f..c404543c 100644 --- a/ext/semian/sliding_window.c +++ b/ext/semian/sliding_window.c @@ -207,9 +207,10 @@ Init_SlidingWindow() rb_define_method(cSlidingWindow, "max_size=", semian_simple_sliding_window_max_size_set, 1); rb_define_method(cSlidingWindow, "values", semian_simple_sliding_window_values, 0); rb_define_method(cSlidingWindow, "last", semian_simple_sliding_window_last, 0); - rb_define_method(cSlidingWindow, "<<", semian_simple_sliding_window_push, 1); + rb_define_method(cSlidingWindow, "push", semian_simple_sliding_window_push, 1); + rb_define_method(cSlidingWindow, "<<", semian_simple_sliding_window_push, 1); // Alias rb_define_method(cSlidingWindow, "clear", semian_simple_sliding_window_clear, 0); - rb_define_method(cSlidingWindow, "destroy", semian_simple_sliding_window_clear, 0); + rb_define_method(cSlidingWindow, "destroy", semian_simple_sliding_window_clear, 0); // Alias rb_define_method(cSlidingWindow, "reject!", semian_simple_sliding_window_reject, 0); } @@ -257,7 +258,7 @@ semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_size, res->parent_key = generate_key(to_s(name)); dprintf("Initializing simple sliding window '%s' (key: %lu)", buffer, res->key); - res->sem_id = initialize_single_semaphore(res->key, SEM_DEFAULT_PERMISSIONS); + res->sem_id = initialize_single_semaphore(res->key, SEM_DEFAULT_PERMISSIONS, 1); res->shmem = get_or_create_shared_memory(res->key, init_fn); res->error_threshold = check_max_size_arg(max_size); res->scale_factor = check_scale_factor_arg(scale_factor); diff --git a/ext/semian/sysv_semaphores.c b/ext/semian/sysv_semaphores.c index 22ede193..b63c7367 100644 --- a/ext/semian/sysv_semaphores.c +++ b/ext/semian/sysv_semaphores.c @@ -122,15 +122,23 @@ perform_semop(int sem_id, short index, short op, short flags, struct timespec *t int get_sem_val(int sem_id, int sem_index) { - dprintf("get_sem_val(sem_id: %d, sem_index: %d)", sem_id, sem_index); int ret = semctl(sem_id, sem_index, GETVAL); if (ret == -1) { rb_raise(eInternal, "error getting value of %s for sem %d, errno: %d (%s)", SEMINDEX_STRING[sem_index], sem_id, errno, strerror(errno)); } - dprintf("get_sem_val(sem_id: %d, sem_index: %d) == %d", sem_id, sem_index, ret); return ret; } +int +set_sem_val(int sem_id, int sem_index, int val) +{ + int ret = semctl(sem_id, sem_index, SETVAL, val); + if (ret == -1) { + rb_raise(eInternal, "error setting value of %s for sem %d, errno: %d (%s)", SEMINDEX_STRING[sem_index], sem_id, errno, strerror(errno)); + } + return ret; +} + void sem_meta_lock(int sem_id) { @@ -169,7 +177,7 @@ acquire_semaphore(void *p) semian_resource_t *res = (semian_resource_t *) p; res->error = 0; res->wait_time = -1; - dprint_sem_vals(res->sem_id); + dprint_sem_vals("acquire_semaphore", res->sem_id); struct timespec begin, end; int benchmark_result = clock_gettime(CLOCK_MONOTONIC, &begin); @@ -196,26 +204,22 @@ initialize_new_semaphore_values(int sem_id, long permissions) if (semctl(sem_id, 0, SETALL, init_vals) == -1) { raise_semian_syscall_error("semctl()", errno); } - dprint_sem_vals(sem_id); + dprint_sem_vals("initialize_new_semaphore_values", sem_id); } static int wait_for_new_semaphore_set(uint64_t key, long permissions) { - int i; - int sem_id = -1; - union semun sem_opts; struct semid_ds sem_ds; - + union semun sem_opts; sem_opts.buf = &sem_ds; - sem_id = semget(key, 1, permissions); + int sem_id = semget(key, 1, permissions); if (sem_id == -1){ raise_semian_syscall_error("semget()", errno); } - for (i = 0; i < ((INTERNAL_TIMEOUT * MICROSECONDS_IN_SECOND) / INIT_WAIT); i++) { - + for (int i = 0; i < ((INTERNAL_TIMEOUT * MICROSECONDS_IN_SECOND) / INIT_WAIT); i++) { if (semctl(sem_id, 0, IPC_STAT, sem_opts) == -1) { raise_semian_syscall_error("semctl()", errno); } @@ -247,9 +251,9 @@ diff_timespec_ms(struct timespec *end, struct timespec *begin) } int -initialize_single_semaphore(uint64_t key, long permissions) +initialize_single_semaphore(uint64_t key, long permissions, int value) { - dprintf("Initializing single semaphore for key:%lu", key); + dprintf("Initializing single semaphore for key:%lu with value %d", key, value); int sem_id = semget(key, 1, IPC_CREAT | IPC_EXCL | permissions); /* @@ -258,26 +262,24 @@ initialize_single_semaphore(uint64_t key, long permissions) */ if (sem_id != -1) { // Happy path - we are the first worker, initialize the semaphore set. - // - if (semctl(sem_id, 0, SETVAL, 1) == -1) { - raise_semian_syscall_error("semctl()", errno); - } dprintf("Created semaphore (key:%lu sem_id:%d)", key, sem_id); + if (semctl(sem_id, 0, SETVAL, value) == -1) { + raise_semian_syscall_error("semctl() failed to set semaphore initial value", errno); + } } else { - // Something went wrong - if (errno != EEXIST) { - raise_semian_syscall_error("semget() failed to initialize semaphore values", errno); - } else { + if (errno == EEXIST) { // The semaphore set already exists, ensure it is initialized sem_id = wait_for_new_semaphore_set(key, permissions); + } else { + raise_semian_syscall_error("semget() failed to initialize semaphore values", errno); } } set_semaphore_permissions(sem_id, permissions); - sem_meta_lock(sem_id); // Sets otime for the first time by acquiring the sem lock - - sem_meta_unlock(sem_id); + // Set otime for the first time by acquiring the sem lock. + sem_meta_unlock(sem_id); // +1 + sem_meta_lock(sem_id); // -1 return sem_id; } diff --git a/ext/semian/sysv_semaphores.h b/ext/semian/sysv_semaphores.h index e170df5e..0049d67b 100644 --- a/ext/semian/sysv_semaphores.h +++ b/ext/semian/sysv_semaphores.h @@ -28,6 +28,8 @@ and functions associated directly weth semops. // 1.9 typedef VALUE (*my_blocking_fn_t)(void*); #define WITHOUT_GVL(fn,a,ubf,b) rb_thread_blocking_region((my_blocking_fn_t)(fn),(a),(ubf),(b)) +#else +#define WITHOUT_GVL(fn,a,ubf,b) #endif // Time to wait for timed ops to complete @@ -94,6 +96,10 @@ perform_semop(int sem_id, short index, short op, short flags, struct timespec *t int get_sem_val(int sem_id, int sem_index); +// Set the current number of tickets in a semaphore by its semaphore index +int +set_sem_val(int sem_id, int sem_index, int val); + // Obtain an exclusive lock on the semaphore set critical section void sem_meta_lock(int sem_id); @@ -113,13 +119,13 @@ acquire_semaphore_without_gvl(void *p); // Initializes a semaphore set with a single semaphore, for general purpose // locking int -initialize_single_semaphore(uint64_t key, long permissions); +initialize_single_semaphore(uint64_t key, long permissions, int value); static inline void -dprint_sem_vals(int sem_id) +dprint_sem_vals(const char *msg, int sem_id) { - dprintf("dprintf(%d)", sem_id); - dprintf("sem_id: %d, lock: %d, tickets: %d configured: %d, registered workers %d", + dprintf("%s (sem_id:%d lock:%d tickets:%d configured:%d, registered_workers:%d)", + msg, sem_id, get_sem_val(sem_id, SI_SEM_LOCK), get_sem_val(sem_id, SI_SEM_TICKETS), diff --git a/ext/semian/tickets.c b/ext/semian/tickets.c index 08d2dbe0..cb4ff6e0 100644 --- a/ext/semian/tickets.c +++ b/ext/semian/tickets.c @@ -49,7 +49,7 @@ update_ticket_count(int sem_id, int tickets) delta = tickets - get_sem_val(sem_id, SI_SEM_CONFIGURED_TICKETS); - dprint_sem_vals(sem_id); + dprint_sem_vals("update_ticket_count", sem_id); if (perform_semop(sem_id, SI_SEM_TICKETS, delta, 0, &ts) == -1) { if (delta < 0 && errno == EAGAIN) { rb_raise(eTimeout, "timeout while trying to update ticket count"); diff --git a/ext/semian/types.h b/ext/semian/types.h index 7935ac5e..116b1dce 100644 --- a/ext/semian/types.h +++ b/ext/semian/types.h @@ -40,27 +40,10 @@ typedef struct { long wait_time; } semian_resource_t; -// Shared circuit breaker structure -typedef struct { -} semian_circuit_breaker_shared_t; - -// Internal circuit breaker structure -typedef struct { - uint64_t key; - int sem_id; - semian_circuit_breaker_shared_t* shmem; -} semian_circuit_breaker_t; - -// Shared simple integer structure -typedef struct { - int val; -} semian_simple_integer_shared_t; - // Internal simple integer structure typedef struct { uint64_t key; int sem_id; - semian_simple_integer_shared_t* shmem; } semian_simple_integer_t; // Shared simple sliding window structure diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index 8791338d..61611967 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -8,8 +8,6 @@ class CircuitBreaker #:nodoc: def initialize(name, exceptions:, success_threshold:, error_threshold:, error_timeout:, implementation:, half_open_resource_timeout: nil, scale_factor: nil) - initialize_circuit_breaker(name, error_threshold) if respond_to?(:initialize_circuit_breaker) - @name = name.to_sym @success_count_threshold = success_threshold @error_count_threshold = error_threshold diff --git a/lib/semian/simple_integer.rb b/lib/semian/simple_integer.rb index e29185d8..8e1b8ff9 100644 --- a/lib/semian/simple_integer.rb +++ b/lib/semian/simple_integer.rb @@ -10,11 +10,17 @@ def initialize(name) reset end + def use_host_circuits + ENV['SEMIAN_CIRCUIT_BREAKER_IMPL'] == 'host' + end + def increment(val = 1) + raise StandardError, "Shouldn't call increment if using host circuits" if use_host_circuits @value += val end def reset + raise StandardError, "Shouldn't call reset if using host circuits" if use_host_circuits @value = 0 end diff --git a/lib/semian/simple_sliding_window.rb b/lib/semian/simple_sliding_window.rb index 3befd0c1..6896e0ca 100644 --- a/lib/semian/simple_sliding_window.rb +++ b/lib/semian/simple_sliding_window.rb @@ -3,8 +3,6 @@ module Semian module Simple class SlidingWindow #:nodoc: - attr_reader :max_size - # A sliding window is a structure that stores the most @max_size recent timestamps # like this: if @max_size = 4, current time is 10, @window =[5,7,9,10]. # Another push of (11) at 11 sec would make @window [7,9,10,11], shifting off 5. @@ -17,23 +15,32 @@ def initialize(name, max_size:, scale_factor: nil) @window = [] end + def use_host_circuits + ENV['SEMIAN_CIRCUIT_BREAKER_IMPL'] == 'host' + end + def size + raise StandardError, "Shouldn't call size if using host circuits" if use_host_circuits @window.size end def last + raise StandardError, "Shouldn't call last if using host circuits" if use_host_circuits @window.last end def values + raise StandardError, "Shouldn't call values if using host circuits" if use_host_circuits @window end def reject!(&block) + raise StandardError, "Shouldn't call reject! if using host circuits" if use_host_circuits @window.reject!(&block) end def push(value) + raise StandardError, "Shouldn't call push if using host circuits" if use_host_circuits resize_to(@max_size - 1) # make room @window << value self @@ -41,12 +48,19 @@ def push(value) alias_method :<<, :push def clear + raise StandardError, "Shouldn't call clear if using host circuits" if use_host_circuits @window.clear self end alias_method :destroy, :clear + def max_size + raise StandardError, "Shouldn't call max_size if using host circuits" if use_host_circuits + @max_size + end + def max_size=(value) + raise StandardError, "Shouldn't call max_size= if using host circuits" if use_host_circuits raise ArgumentError, "max_size must be positive" if value <= 0 @max_size = value resize_to(value) @@ -55,6 +69,7 @@ def max_size=(value) private def resize_to(size) + raise StandardError, "Shouldn't call resize_to if using host circuits" if use_host_circuits @window = @window.last(size) if @window.size >= size end end diff --git a/test/simple_integer_test.rb b/test/simple_integer_test.rb index 925c425f..aaeec381 100644 --- a/test/simple_integer_test.rb +++ b/test/simple_integer_test.rb @@ -7,7 +7,7 @@ def setup end def teardown - @integer.destroy + @integer.destroy unless @integer.nil? end module IntegerTestCases @@ -15,7 +15,7 @@ def test_access_value assert_equal(0, @integer.value) @integer.value = 99 assert_equal(99, @integer.value) - time_now = (Time.now).to_i + time_now = (Time.now).to_i % (2**15) @integer.value = time_now assert_equal(time_now, @integer.value) @integer.value = 6 @@ -43,25 +43,41 @@ def test_reset assert_equal(0, @integer.value) end + def assert_equal_with_retry(expected) + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + while Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time < 1.0 + return true if expected == yield + sleep(0.1) + end + + assert_equal(expected, yield) + end + # Without locks, this only passes around 1 in every 5 runs def test_increment_race - process_count = 255 - 100.times do + process_count = 16 + 10.times do + pids = [] + + id = Time.now.strftime('%H:%M:%S.%N') + @integer = ::Semian::ThreadSafe::Integer.new(id) + process_count.times do - fork do - value = @integer.increment(1) - exit!(value) + pids << fork do + @integer.increment(1) + sleep(60) end end - exit_codes = Process.waitall.map { |_, status| status.exitstatus } if ENV['SEMIAN_CIRCUIT_BREAKER_IMPL'] == 'host' - # Host-based circuits: No two processes should exit with the same exit code - assert_equal(process_count, exit_codes.uniq.length) + # Host-based circuits: Forked processes should increment the host integer. + assert_equal_with_retry(process_count) { @integer.value } else - # Worker-based circuits: All the processes should exit with the same code - assert_equal(1, exit_codes.uniq.length) + # Worker-based circuits: Forked processes should increment the worker integer. + assert_equal_with_retry(0) { @integer.value } end + ensure + pids.each { |pid| Process.kill('TERM', pid) } end end end diff --git a/test/simple_sliding_window_test.rb b/test/simple_sliding_window_test.rb index 24de1d31..99a7c06e 100644 --- a/test/simple_sliding_window_test.rb +++ b/test/simple_sliding_window_test.rb @@ -7,6 +7,10 @@ def setup @sliding_window.clear end + def teardown + @sliding_window.destroy unless @sliding_window.nil? + end + def test_clear id = Time.now.strftime('%H:%M:%S.%N') window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 6) @@ -218,6 +222,12 @@ def test_scale_factor end end + def test_max_size + 8192.times do |i| + @sliding_window << i + end + end + private def assert_sliding_window(sliding_window, array, max_size) diff --git a/test/test_helper.rb b/test/test_helper.rb index ff93af5f..4adfc30b 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,4 +1,5 @@ require 'minitest/autorun' +require 'minitest/reporters' require 'semian' require 'semian/mysql2' require 'semian/redis' @@ -63,6 +64,8 @@ class Minitest::Test include BackgroundHelper end +Minitest::Reporters.use! [Minitest::Reporters::SpecReporter.new()] + def flaky ENV["SKIP_FLAKY_TESTS"] end From 4682bede33b4392fc42b074c4ace978ee2a38be1 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Tue, 2 Jul 2019 14:49:04 -0400 Subject: [PATCH 39/49] Print debug messages with timestamp and process ID --- ext/semian/resource.c | 5 ++--- ext/semian/util.h | 7 ++++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ext/semian/resource.c b/ext/semian/resource.c index afd448df..bbd438c6 100644 --- a/ext/semian/resource.c +++ b/ext/semian/resource.c @@ -117,13 +117,12 @@ semian_resource_reset_workers(VALUE self) VALUE semian_resource_unregister_worker(VALUE self) { - int ret; semian_resource_t *res = NULL; - TypedData_Get_Struct(self, semian_resource_t, &semian_resource_type, res); sem_meta_lock(res->sem_id); - ret = perform_semop(res->sem_id, SI_SEM_REGISTERED_WORKERS, -1, IPC_NOWAIT | SEM_UNDO, NULL); + dprintf("Unregistering worker for sem_id:%d", res->sem_id); + int ret = perform_semop(res->sem_id, SI_SEM_REGISTERED_WORKERS, -1, IPC_NOWAIT | SEM_UNDO, NULL); sem_meta_unlock(res->sem_id); if ( ret == -1) { diff --git a/ext/semian/util.h b/ext/semian/util.h index 242a4a09..8a2470bc 100644 --- a/ext/semian/util.h +++ b/ext/semian/util.h @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -16,7 +17,11 @@ #define dprintf(fmt, ...) \ do { \ if (DEBUG_TEST) { \ - printf("[DEBUG] %s:%d - " fmt "\n", __FILE__, __LINE__, ##__VA_ARGS__); \ + const pid_t pid = getpid(); \ + struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); \ + struct tm t; localtime_r(&(ts.tv_sec), &t); \ + char buf[128]; strftime(buf, sizeof(buf), "%H:%M:%S", &t); \ + printf("%s.%ld [DEBUG] (%d): %s:%d - " fmt "\n", buf, ts.tv_nsec, pid, __FILE__, __LINE__, ##__VA_ARGS__); \ } \ } while (0) From f43ccbfe214fecef17d46f830b30ca5e1216bc2b Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Wed, 3 Jul 2019 14:49:33 +0000 Subject: [PATCH 40/49] Tests are too noisy --- Rakefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Rakefile b/Rakefile index 26cce7ee..aca13d6b 100644 --- a/Rakefile +++ b/Rakefile @@ -41,8 +41,6 @@ require 'rake/testtask' Rake::TestTask.new 'test' do |t| t.libs = %w(lib test) t.pattern = "test/*_test.rb" - t.verbose = true - t.warning = true end # ========================================================== From af854a5a824a177d529c065921c826cb25d6eb11 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Wed, 3 Jul 2019 14:50:16 +0000 Subject: [PATCH 41/49] Don't force sliding window entries to be monotonic --- ext/semian/sliding_window.c | 99 +++++++++++++++++------------- ext/semian/types.h | 1 - lib/semian/circuit_breaker.rb | 1 + lib/semian/protected_resource.rb | 2 +- lib/semian/resource.rb | 12 ++++ lib/semian/unprotected_resource.rb | 12 ++++ test/circuit_breaker_test.rb | 30 +++++++++ test/resource_test.rb | 4 +- test/simple_sliding_window_test.rb | 10 +-- 9 files changed, 119 insertions(+), 52 deletions(-) diff --git a/ext/semian/sliding_window.c b/ext/semian/sliding_window.c index c404543c..f85109cc 100644 --- a/ext/semian/sliding_window.c +++ b/ext/semian/sliding_window.c @@ -34,7 +34,6 @@ static void init_fn(void* ptr) res->max_size = 0; res->length = 0; res->start = 0; - res->end = 0; } static int @@ -92,16 +91,17 @@ check_scale_factor_arg(VALUE scale_factor) } static VALUE -grow_window(semian_simple_sliding_window_shared_t* window, int new_max_size) +grow_window(int sem_id, semian_simple_sliding_window_shared_t* window, int new_max_size) { if (new_max_size > SLIDING_WINDOW_MAX_SIZE) return Qnil; + int end = window->max_size ? (window->start + window->length) % window->max_size : 0; + dprintf("Growing window - sem_id:%d start:%d end:%d length:%d max_size:%d new_max_size:%d", sem_id, window->start, end, window->length, window->max_size, new_max_size); + if (window->length == 0) { window->start = 0; - window->end = 0; - } else if (window->end > window->start) { + } else if (end > window->start) { // Easy case - the window doesn't wrap around - window->end = window->start + window->length; } else { // Hard case - the window wraps, and data might need to move int offset = new_max_size - window->max_size; @@ -125,24 +125,25 @@ static void swap(int *a, int *b) { } static VALUE -shrink_window(semian_simple_sliding_window_shared_t* window, int new_max_size) +shrink_window(int sem_id, semian_simple_sliding_window_shared_t* window, int new_max_size) { if (new_max_size > SLIDING_WINDOW_MAX_SIZE) return Qnil; int new_length = (new_max_size > window->length) ? window->length : new_max_size; - dprintf("Shrinking window - start:%d end:%d length:%d max_size:%d", window->start, window->end, window->length, window->max_size); + int end = window->max_size ? (window->start + window->length) % window->max_size : 0; + dprintf("Shrinking window - sem_id:%d start:%d end:%d length:%d max_size:%d new_max_size:%d", sem_id, window->start, end, window->length, window->max_size, new_max_size); + if (window->length == 0) { window->start = 0; - window->end = 0; - } else if (window->end > window->start) { + } else if (end > window->start) { // Easy case - the window doesn't wrap around window->start = window->start + new_length; } else { // Hard case - the window wraps, so re-index the data // Adapted from http://www.cplusplus.com/reference/algorithm/rotate/ int first = 0; - int middle = (window->end - new_max_size + window->max_size) % window->max_size; + int middle = (end - new_max_size + window->max_size) % window->max_size; int last = window->max_size; int next = middle; while (first != next) { @@ -154,7 +155,6 @@ shrink_window(semian_simple_sliding_window_shared_t* window, int new_max_size) } } window->start = 0; - window->end = new_length; } window->max_size = new_max_size; @@ -164,18 +164,14 @@ shrink_window(semian_simple_sliding_window_shared_t* window, int new_max_size) } static VALUE -resize_window(semian_simple_sliding_window_shared_t* window, int new_max_size) +resize_window(int sem_id, semian_simple_sliding_window_shared_t* window, int new_max_size) { if (new_max_size > SLIDING_WINDOW_MAX_SIZE) return Qnil; if (window->max_size < new_max_size) { - dprintf("Growing window to %d", new_max_size); - return grow_window(window, new_max_size); + return grow_window(sem_id, window, new_max_size); } else if (window->max_size > new_max_size) { - dprintf("Shrinking window to %d", new_max_size); - return shrink_window(window, new_max_size); - } else { - dprintf("Not re-sizing window"); + return shrink_window(sem_id, window, new_max_size); } return Qnil; @@ -202,6 +198,7 @@ Init_SlidingWindow() rb_define_alloc_func(cSlidingWindow, semian_simple_sliding_window_alloc); rb_define_method(cSlidingWindow, "initialize_sliding_window", semian_simple_sliding_window_initialize, 3); rb_define_method(cSlidingWindow, "size", semian_simple_sliding_window_size, 0); + rb_define_method(cSlidingWindow, "length", semian_simple_sliding_window_size, 0); // Alias rb_define_method(cSlidingWindow, "resize_to", semian_simple_sliding_window_resize_to, 1); rb_define_method(cSlidingWindow, "max_size", semian_simple_sliding_window_max_size_get, 0); rb_define_method(cSlidingWindow, "max_size=", semian_simple_sliding_window_max_size_set, 1); @@ -270,7 +267,7 @@ semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_size, int error_threshold = max(res->error_threshold, (int) ceil(workers * scale * res->error_threshold)); dprintf(" workers:%d scale:%0.2f error_threshold:%d", workers, scale, error_threshold); - resize_window(res->shmem, error_threshold); + resize_window(res->sem_id, res->shmem, error_threshold); } sem_meta_unlock(res->sem_id); @@ -305,7 +302,7 @@ semian_simple_sliding_window_resize_to(VALUE self, VALUE new_size) sem_meta_lock(res->sem_id); { - retval = resize_window(res->shmem, new_max_size); + retval = resize_window(res->sem_id, res->shmem, new_max_size); } sem_meta_unlock(res->sem_id); @@ -340,7 +337,7 @@ semian_simple_sliding_window_max_size_set(VALUE self, VALUE new_size) sem_meta_lock(res->sem_id); { - retval = resize_window(res->shmem, new_max_size); + retval = resize_window(res->sem_id, res->shmem, new_max_size); } sem_meta_unlock(res->sem_id); @@ -393,13 +390,25 @@ semian_simple_sliding_window_clear(VALUE self) dprintf("Clearing sliding window"); res->shmem->length = 0; res->shmem->start = 0; - res->shmem->end = 0; } sem_meta_unlock(res->sem_id); return self; } +// Handy for debugging the sliding window, but too noisy for regular debugging. +/* +static void dprint_window(semian_simple_sliding_window_shared_t *window) +{ + dprintf("---"); + for (int i = 0; i < window->length; ++i) { + const int index = (window->start + i) % window->max_size; + dprintf(" %0d: data[%d] = %d", i, index, window->data[index]); + } + dprintf("---"); +} +*/ + VALUE semian_simple_sliding_window_reject(VALUE self) { @@ -409,24 +418,30 @@ semian_simple_sliding_window_reject(VALUE self) sem_meta_lock(res->sem_id); { - // Store these values because we're going to be modifying the buffer. - int start = res->shmem->start; - int length = res->shmem->length; - dprintf("reject! - start:%d end:%d length:%d max_size:%d", res->shmem->start, res->shmem->end, res->shmem->length, res->shmem->max_size); - - int cleared = 0; - for (int i = 0; i < length; ++i) { - int index = (start + i) % length; - int value = res->shmem->data[index]; - VALUE y = rb_yield(RB_INT2NUM(value)); - if (RTEST(y)) { - if (cleared++ != i) { - sem_meta_unlock(res->sem_id); - rb_raise(rb_eArgError, "reject! must delete monotonically"); + semian_simple_sliding_window_shared_t *window = res->shmem; + const int start = window->start; + const int length = window->length; + const int max_size = window->max_size; + + if (max_size && length) { + int wptr = (start + length + max_size - 1) % max_size; + + dprintf("Before reject! start:%d length:%d max_size:%d", window->start, window->length, window->max_size); + for (int i = 0; i < length; ++i) { + const int rptr = (start + length + max_size - i - 1) % max_size; + + const int value = window->data[rptr]; + if (RTEST(rb_yield(RB_INT2NUM(value)))) { + window->length--; + window->data[wptr] = value; + } else { + window->data[wptr] = value; + wptr = (wptr + max_size - 1) % max_size; } - res->shmem->start = (res->shmem->start + 1) % res->shmem->length; - res->shmem->length--; } + + window->start = (wptr + 1) % max_size; + dprintf("After reject! start:%d length:%d max_size:%d", window->start, window->length, window->max_size); } } sem_meta_unlock(res->sem_id); @@ -441,7 +456,7 @@ semian_simple_sliding_window_push(VALUE self, VALUE value) sem_meta_lock(res->sem_id); { - dprintf("Before: start:%d end:%d length:%d max_size:%d", res->shmem->start, res->shmem->end, res->shmem->length, res->shmem->max_size); + dprintf("Before: start:%d length:%d max_size:%d", res->shmem->start, res->shmem->length, res->shmem->max_size); // If the window is full, make room by popping off the front. if (res->shmem->length == res->shmem->max_size) { res->shmem->length--; @@ -449,10 +464,10 @@ semian_simple_sliding_window_push(VALUE self, VALUE value) } // Push onto the back of the window. + int index = (res->shmem->start + res->shmem->length) % res->shmem->max_size; res->shmem->length++; - res->shmem->data[res->shmem->end] = RB_NUM2INT(value); - dprintf("Pushed %d onto data[%d] (length %d)", RB_NUM2INT(value), res->shmem->end, res->shmem->length); - res->shmem->end = (res->shmem->end + 1) % res->shmem->max_size; + res->shmem->data[index] = RB_NUM2INT(value); + dprintf("Pushed %d onto data[%d] (length %d)", RB_NUM2INT(value), index, res->shmem->length); } sem_meta_unlock(res->sem_id); diff --git a/ext/semian/types.h b/ext/semian/types.h index 116b1dce..b6a10a72 100644 --- a/ext/semian/types.h +++ b/ext/semian/types.h @@ -51,7 +51,6 @@ typedef struct { int max_size; int length; int start; - int end; int data[SLIDING_WINDOW_MAX_SIZE]; } semian_simple_sliding_window_shared_t; diff --git a/lib/semian/circuit_breaker.rb b/lib/semian/circuit_breaker.rb index 61611967..788035ca 100644 --- a/lib/semian/circuit_breaker.rb +++ b/lib/semian/circuit_breaker.rb @@ -3,6 +3,7 @@ class CircuitBreaker #:nodoc: extend Forwardable def_delegators :@state, :closed?, :open?, :half_open? + def_delegators :@errors, :size, :max_size, :values attr_reader :name, :half_open_resource_timeout, :error_timeout, :state, :last_error diff --git a/lib/semian/protected_resource.rb b/lib/semian/protected_resource.rb index 73c10fac..5c032083 100644 --- a/lib/semian/protected_resource.rb +++ b/lib/semian/protected_resource.rb @@ -4,7 +4,7 @@ class ProtectedResource def_delegators :@bulkhead, :destroy, :count, :semid, :tickets, :registered_workers def_delegators :@circuit_breaker, :reset, :mark_failed, :mark_success, :request_allowed?, - :open?, :closed?, :half_open? + :open?, :closed?, :half_open?, :size, :max_size, :values attr_reader :bulkhead, :circuit_breaker, :name attr_accessor :updated_at diff --git a/lib/semian/resource.rb b/lib/semian/resource.rb index 1dd72658..0505bfa8 100644 --- a/lib/semian/resource.rb +++ b/lib/semian/resource.rb @@ -36,6 +36,18 @@ def count 0 end + def size + 0 + end + + def max_size + 0 + end + + def values + [] + end + def tickets 0 end diff --git a/lib/semian/unprotected_resource.rb b/lib/semian/unprotected_resource.rb index 8a88b826..dd2c3d3a 100644 --- a/lib/semian/unprotected_resource.rb +++ b/lib/semian/unprotected_resource.rb @@ -29,6 +29,18 @@ def count 0 end + def size + 0 + end + + def max_size + 0 + end + + def values + [] + end + def semid 0 end diff --git a/test/circuit_breaker_test.rb b/test/circuit_breaker_test.rb index 8d4275bf..d97e4d9d 100644 --- a/test/circuit_breaker_test.rb +++ b/test/circuit_breaker_test.rb @@ -14,6 +14,35 @@ def setup end Semian.register(id, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) @resource = Semian[id] + @resource.reset + end + + def test_destroy + id = Time.now.strftime('%H:%M:%S.%N') + + # Create the resource and check that it was reset. + Semian.register(id, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) + resource = Semian[id] + assert_equal(0, resource.size) + assert_equal(2, resource.max_size) + assert_equal([], resource.values) + + # Open the circuit. + open_circuit!(resource, 2) + assert_equal(2, resource.size) + assert_equal(2, resource.max_size) + + # Destroy the resource and check that it was destroyed. + Semian.destroy(id) + resource = Semian[id] + assert_nil(resource, "Resource was not destroyed") + + # Re-create the resource and check that it was reset. + Semian.register(id, tickets: 1, exceptions: [SomeError], error_threshold: 2, error_timeout: 5, success_threshold: 1) + resource = Semian[id] + assert_equal(0, resource.size) + assert_equal(2, resource.max_size) + assert_equal([], resource.values) end def test_acquire_yield_when_the_circuit_is_closed @@ -41,6 +70,7 @@ def test_after_error_timeout_is_elapsed_requests_are_attempted_again end def test_until_success_threshold_is_reached_a_single_error_will_reopen_the_circuit + assert_equal(0, @resource.size) half_open_cicuit! trigger_error! assert_circuit_opened diff --git a/test/resource_test.rb b/test/resource_test.rb index 79dd7885..769f6364 100644 --- a/test/resource_test.rb +++ b/test/resource_test.rb @@ -380,10 +380,12 @@ def test_sem_undo end end + # TODO(michaelkipper): Shouldn't need to rescue InternalError, this test + # should deterministically throw SyscallError. def test_destroy resource = create_resource :testing, tickets: 1 resource.destroy - assert_raises Semian::SyscallError do + assert_raises(Semian::InternalError, Semian::SyscallError) do resource.acquire {} end end diff --git a/test/simple_sliding_window_test.rb b/test/simple_sliding_window_test.rb index 99a7c06e..56813bcb 100644 --- a/test/simple_sliding_window_test.rb +++ b/test/simple_sliding_window_test.rb @@ -51,16 +51,12 @@ def test_sliding_window_reject assert_sliding_window(@sliding_window, [4, 5, 6, 7], 6) end - def test_sliding_window_reject_failure - skip unless ENV['SEMIAN_CIRCUIT_BREAKER_IMPL'] == 'host' - + def test_sliding_window_reject_non_contiguous @sliding_window << 0 << 1 << 2 << 3 << 4 << 5 << 6 << 7 assert_equal(6, @sliding_window.size) assert_sliding_window(@sliding_window, [2, 3, 4, 5, 6, 7], 6) - assert_raises ArgumentError do - # This deletes from the middle of the array - @sliding_window.reject! { |val| val == 3 } - end + @sliding_window.reject! { |val| val == 3 } + assert_sliding_window(@sliding_window, [2, 4, 5, 6, 7], 6) end def test_resize_to_less_than_1_raises From 6445222584bbb6c38b856b59876a210388ff2ed1 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Wed, 3 Jul 2019 15:19:49 -0400 Subject: [PATCH 42/49] Review comments --- ext/semian/sliding_window.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ext/semian/sliding_window.c b/ext/semian/sliding_window.c index f85109cc..17564bbc 100644 --- a/ext/semian/sliding_window.c +++ b/ext/semian/sliding_window.c @@ -426,6 +426,17 @@ semian_simple_sliding_window_reject(VALUE self) if (max_size && length) { int wptr = (start + length + max_size - 1) % max_size; + // Walk the sliding window backward, from the last element to the first, + // pushing the entries to the back of the ring. When we've gone through + // every element, set the start pointer to the new location. + // + // Example, deleting "2": + // S E S E + // [x,x,0,1,2,3,x,x] --> [x,x,x,0,1,3,x,x] + // 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 + // + // The runtime of this algorithm is theta(n), but n tends to be small. + // dprintf("Before reject! start:%d length:%d max_size:%d", window->start, window->length, window->max_size); for (int i = 0; i < length; ++i) { const int rptr = (start + length + max_size - i - 1) % max_size; @@ -433,7 +444,6 @@ semian_simple_sliding_window_reject(VALUE self) const int value = window->data[rptr]; if (RTEST(rb_yield(RB_INT2NUM(value)))) { window->length--; - window->data[wptr] = value; } else { window->data[wptr] = value; wptr = (wptr + max_size - 1) % max_size; From 3655ae82c42254d94bc84f810fc210e7ad5ce242 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Mon, 1 Jul 2019 16:03:27 -0400 Subject: [PATCH 43/49] Allow SEMIAN_CIRCUIT_BREAKER_FORCE_HOST to force-enable host-based circuits --- ext/semian/semian.c | 55 ++++++++++++++++++++++++++++++++++- ext/semian/test/Makefile | 32 ++++++++++++++++++++ ext/semian/test/semian_test.c | 37 +++++++++++++++++++++++ 3 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 ext/semian/test/Makefile create mode 100644 ext/semian/test/semian_test.c diff --git a/ext/semian/semian.c b/ext/semian/semian.c index cbabfb7b..85fd6e69 100644 --- a/ext/semian/semian.c +++ b/ext/semian/semian.c @@ -73,9 +73,62 @@ void Init_semian() } } +static const char* +get_hostname() +{ + const char *VARIABLES[] = { + "KUBE_HOSTNAME", + "KUBE_HOST_NAME", + "KUBE_NODENAME", + "KUBE_NODE_NAME", + "NODENAME", + "NODE_NAME", + "HOSTNAME", + "HOST_NAME", + }; + + for (size_t i = 0; i < sizeof(VARIABLES); ++i) { + const char *hostname = getenv(VARIABLES[i]); + if (hostname != NULL) { + dprintf("Host name is %s", hostname); + return hostname; + } + } + + return NULL; +} + +static int +force_c_circuits() +{ + const char *force_host = getenv("SEMIAN_CIRCUIT_BREAKER_FORCE_HOST"); + if (!force_host) return 0; + const char *hostname = get_hostname(); + if (!hostname) return 0; + + int retval = 0; + + char *hostnames = strdup(force_host); + + char *cursor = NULL; + char *saveptr = hostnames; + while ((cursor = strtok_r(saveptr, ",; ", &saveptr))) { + if (strcmp(hostname, cursor) == 0) { + fprintf(stderr, "Info: Semian forcing host-based circuit implementation\n"); + retval = 1; + break; + } + } + + free(hostnames); + return retval; +} + static int use_c_circuits() { - char *circuit_impl = getenv("SEMIAN_CIRCUIT_BREAKER_IMPL"); + if (force_c_circuits()) return 1; + + const char *circuit_impl = getenv("SEMIAN_CIRCUIT_BREAKER_IMPL"); if (circuit_impl == NULL) { fprintf(stderr, "Warning: Defaulting to Semian worker-based circuit breaker implementation\n"); return 0; diff --git a/ext/semian/test/Makefile b/ext/semian/test/Makefile new file mode 100644 index 00000000..d84591c2 --- /dev/null +++ b/ext/semian/test/Makefile @@ -0,0 +1,32 @@ +# Makefile + +# 'gcc -I. -I/usr/local/include/ruby-2.6.0/x86_64-linux -I/usr/local/include/ruby-2.6.0/ruby/backward \ +# -I/usr/local/include/ruby-2.6.0 -I../../../../ext/semian \ +# -DHAVE_SYS_IPC_H -DHAVE_SYS_SEM_H -DHAVE_SYS_TYPES_H -DHAVE_RB_THREAD_CALL_WITHOUT_GVL \ +# -fPIC -D_GNU_SOURCE -Werror -Wall -std=gnu99 -O3 -o semian.o -c ../../../../ext/semian/semian.c + +CC := gcc +LIBRARIES := ruby c m crypto +WARNINGS := error all +DEFINES := HAVE_SYS_IPC_H HAVE_SYS_SEM_H HAVE_SYS_TYPES_H HAVE_RB_THREAD_CALL_WITHOUT_GVL DEBUG _GNU_SOURCE +INCLUDES := . /usr/local/include/ruby-2.6.0/ruby/backward /usr/local/include/ruby-2.6.0 /usr/local/include/ruby-2.6.0 /usr/local/include/ruby-2.6.0/x86_64-linux +CFLAGS := $(patsubst %,-l%,$(LIBRARIES)) $(patsubst %,-W%,$(WARNINGS)) $(patsubst %,-D%,$(DEFINES)) $(patsubst %,-I%,$(INCLUDES)) -std=gnu99 -O0 -g +ROOTDIR := /workspace +OBJDIR := $(ROOTDIR)/tmp/x86_64-linux/semian/2.6.3 +OBJS := $(wildcard $(OBJDIR)/*.o) +TESTS := semian_test + +all : $(TESTS) + +%_test.o : %_test.c $(filter-out $(OBJDIR)/semian.o,$(OBJS)) + $(CC) -o $@ $^ $(CFLAGS) + +%_test : %_test.o + echo "Info: Running $@" + ./$< + +clean : + rm -f *.o + +.SECONDARY : $(patsubst %,%.o,$(TESTS)) +.PHONY : all clean diff --git a/ext/semian/test/semian_test.c b/ext/semian/test/semian_test.c new file mode 100644 index 00000000..b7820812 --- /dev/null +++ b/ext/semian/test/semian_test.c @@ -0,0 +1,37 @@ +#include +#include + +// Include the C source file, so we can access static functions. +#include "../semian.c" + +void assert_equal_int(int expected, int actual) +{ + if (expected != actual) { + printf("Error: %d != %d\n", expected, actual); + exit(EXIT_FAILURE); + } +} + +void test_force_c_circuits() +{ + setenv("KUBE_HOSTNAME", "machine-1", 1); + + setenv("SEMIAN_CIRCUIT_BREAKER_FORCE_HOST", "machine-2,machine-3", 1); + assert_equal_int(0, force_c_circuits()); + + setenv("SEMIAN_CIRCUIT_BREAKER_FORCE_HOST", "machine-1,machine-2,machine-3", 1); + assert_equal_int(1, force_c_circuits()); +} + +int main(int argc, char **argv) +{ + printf("Info: Running Semian test\n"); + + // Ruby will replace functions like strdup() with ruby_strdup(), which will + // segfault if Ruby is not ready. Initialize Ruby so we don't have problems. + ruby_init(); + + test_force_c_circuits(); + + return EXIT_SUCCESS; +} From 926074d34631c4e4fcafd808ca81de714ce3d72d Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Tue, 2 Jul 2019 16:18:43 -0400 Subject: [PATCH 44/49] Build semian libraries before unit tests --- ext/semian/test/Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ext/semian/test/Makefile b/ext/semian/test/Makefile index d84591c2..62084369 100644 --- a/ext/semian/test/Makefile +++ b/ext/semian/test/Makefile @@ -13,10 +13,13 @@ INCLUDES := . /usr/local/include/ruby-2.6.0/ruby/backward /usr/local/include/rub CFLAGS := $(patsubst %,-l%,$(LIBRARIES)) $(patsubst %,-W%,$(WARNINGS)) $(patsubst %,-D%,$(DEFINES)) $(patsubst %,-I%,$(INCLUDES)) -std=gnu99 -O0 -g ROOTDIR := /workspace OBJDIR := $(ROOTDIR)/tmp/x86_64-linux/semian/2.6.3 -OBJS := $(wildcard $(OBJDIR)/*.o) +OBJS = $(wildcard $(OBJDIR)/*.o) TESTS := semian_test -all : $(TESTS) +all : libs $(TESTS) + +libs : + cd $(ROOTDIR) && SEMIAN_DEBUG=1 bundle exec rake build %_test.o : %_test.c $(filter-out $(OBJDIR)/semian.o,$(OBJS)) $(CC) -o $@ $^ $(CFLAGS) From 05796e7564b4fd77d5f68cbc80348dcce3538bec Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Wed, 3 Jul 2019 17:00:22 -0400 Subject: [PATCH 45/49] Force circuits in Ruby --- ext/semian/semian.c | 55 +---------------------------------- ext/semian/test/Makefile | 35 ---------------------- ext/semian/test/semian_test.c | 37 ----------------------- lib/semian.rb | 12 ++++++++ test/semian_test.rb | 11 +++++++ 5 files changed, 24 insertions(+), 126 deletions(-) delete mode 100644 ext/semian/test/Makefile delete mode 100644 ext/semian/test/semian_test.c diff --git a/ext/semian/semian.c b/ext/semian/semian.c index 85fd6e69..cbabfb7b 100644 --- a/ext/semian/semian.c +++ b/ext/semian/semian.c @@ -73,62 +73,9 @@ void Init_semian() } } -static const char* -get_hostname() -{ - const char *VARIABLES[] = { - "KUBE_HOSTNAME", - "KUBE_HOST_NAME", - "KUBE_NODENAME", - "KUBE_NODE_NAME", - "NODENAME", - "NODE_NAME", - "HOSTNAME", - "HOST_NAME", - }; - - for (size_t i = 0; i < sizeof(VARIABLES); ++i) { - const char *hostname = getenv(VARIABLES[i]); - if (hostname != NULL) { - dprintf("Host name is %s", hostname); - return hostname; - } - } - - return NULL; -} - -static int -force_c_circuits() -{ - const char *force_host = getenv("SEMIAN_CIRCUIT_BREAKER_FORCE_HOST"); - if (!force_host) return 0; - const char *hostname = get_hostname(); - if (!hostname) return 0; - - int retval = 0; - - char *hostnames = strdup(force_host); - - char *cursor = NULL; - char *saveptr = hostnames; - while ((cursor = strtok_r(saveptr, ",; ", &saveptr))) { - if (strcmp(hostname, cursor) == 0) { - fprintf(stderr, "Info: Semian forcing host-based circuit implementation\n"); - retval = 1; - break; - } - } - - free(hostnames); - return retval; -} - static int use_c_circuits() { - if (force_c_circuits()) return 1; - - const char *circuit_impl = getenv("SEMIAN_CIRCUIT_BREAKER_IMPL"); + char *circuit_impl = getenv("SEMIAN_CIRCUIT_BREAKER_IMPL"); if (circuit_impl == NULL) { fprintf(stderr, "Warning: Defaulting to Semian worker-based circuit breaker implementation\n"); return 0; diff --git a/ext/semian/test/Makefile b/ext/semian/test/Makefile deleted file mode 100644 index 62084369..00000000 --- a/ext/semian/test/Makefile +++ /dev/null @@ -1,35 +0,0 @@ -# Makefile - -# 'gcc -I. -I/usr/local/include/ruby-2.6.0/x86_64-linux -I/usr/local/include/ruby-2.6.0/ruby/backward \ -# -I/usr/local/include/ruby-2.6.0 -I../../../../ext/semian \ -# -DHAVE_SYS_IPC_H -DHAVE_SYS_SEM_H -DHAVE_SYS_TYPES_H -DHAVE_RB_THREAD_CALL_WITHOUT_GVL \ -# -fPIC -D_GNU_SOURCE -Werror -Wall -std=gnu99 -O3 -o semian.o -c ../../../../ext/semian/semian.c - -CC := gcc -LIBRARIES := ruby c m crypto -WARNINGS := error all -DEFINES := HAVE_SYS_IPC_H HAVE_SYS_SEM_H HAVE_SYS_TYPES_H HAVE_RB_THREAD_CALL_WITHOUT_GVL DEBUG _GNU_SOURCE -INCLUDES := . /usr/local/include/ruby-2.6.0/ruby/backward /usr/local/include/ruby-2.6.0 /usr/local/include/ruby-2.6.0 /usr/local/include/ruby-2.6.0/x86_64-linux -CFLAGS := $(patsubst %,-l%,$(LIBRARIES)) $(patsubst %,-W%,$(WARNINGS)) $(patsubst %,-D%,$(DEFINES)) $(patsubst %,-I%,$(INCLUDES)) -std=gnu99 -O0 -g -ROOTDIR := /workspace -OBJDIR := $(ROOTDIR)/tmp/x86_64-linux/semian/2.6.3 -OBJS = $(wildcard $(OBJDIR)/*.o) -TESTS := semian_test - -all : libs $(TESTS) - -libs : - cd $(ROOTDIR) && SEMIAN_DEBUG=1 bundle exec rake build - -%_test.o : %_test.c $(filter-out $(OBJDIR)/semian.o,$(OBJS)) - $(CC) -o $@ $^ $(CFLAGS) - -%_test : %_test.o - echo "Info: Running $@" - ./$< - -clean : - rm -f *.o - -.SECONDARY : $(patsubst %,%.o,$(TESTS)) -.PHONY : all clean diff --git a/ext/semian/test/semian_test.c b/ext/semian/test/semian_test.c deleted file mode 100644 index b7820812..00000000 --- a/ext/semian/test/semian_test.c +++ /dev/null @@ -1,37 +0,0 @@ -#include -#include - -// Include the C source file, so we can access static functions. -#include "../semian.c" - -void assert_equal_int(int expected, int actual) -{ - if (expected != actual) { - printf("Error: %d != %d\n", expected, actual); - exit(EXIT_FAILURE); - } -} - -void test_force_c_circuits() -{ - setenv("KUBE_HOSTNAME", "machine-1", 1); - - setenv("SEMIAN_CIRCUIT_BREAKER_FORCE_HOST", "machine-2,machine-3", 1); - assert_equal_int(0, force_c_circuits()); - - setenv("SEMIAN_CIRCUIT_BREAKER_FORCE_HOST", "machine-1,machine-2,machine-3", 1); - assert_equal_int(1, force_c_circuits()); -} - -int main(int argc, char **argv) -{ - printf("Info: Running Semian test\n"); - - // Ruby will replace functions like strdup() with ruby_strdup(), which will - // segfault if Ruby is not ready. Initialize Ruby so we don't have problems. - ruby_init(); - - test_force_c_circuits(); - - return EXIT_SUCCESS; -} diff --git a/lib/semian.rb b/lib/semian.rb index d579f774..c59f636a 100644 --- a/lib/semian.rb +++ b/lib/semian.rb @@ -296,7 +296,19 @@ def require_keys!(required = [], **options) end end +def hostname + v = %w(KUBE_HOSTNAME KUBE_HOST_NAME KUBE_NODENAME KUBE_NODE_NAME NODENAME NODE_NAME HOSTNAME HOST_NAME) + var = v.filter { |x| ENV.include?(x) }.first + ENV[var] if var +end + +def force_host_circuits + return false unless ENV.include?('SEMIAN_CIRCUIT_BREAKER_FORCE_HOST') + ENV['SEMIAN_CIRCUIT_BREAKER_FORCE_HOST'].split(',').include?(hostname) +end + if Semian.semaphores_enabled? + ENV['SEMIAN_CIRCUIT_BREAKER_IMPL'] = 'host' if force_host_circuits require 'semian/semian' else Semian::MAX_TICKETS = 0 diff --git a/test/semian_test.rb b/test/semian_test.rb index 65d6958f..568ffcb2 100644 --- a/test/semian_test.rb +++ b/test/semian_test.rb @@ -91,4 +91,15 @@ def test_disabled_via_semian_wide_env_var ensure ENV.delete('SEMIAN_DISABLED') end + + def test_force_host_circuits + refute force_host_circuits + ENV['SEMIAN_CIRCUIT_BREAKER_FORCE_HOST'] = 'machine-1,machine-2,machine-3' + refute force_host_circuits + ENV['KUBE_HOSTNAME'] = 'machine-2' + assert force_host_circuits + ensure + ENV.delete('SEMIAN_CIRCUIT_BREAKER_FORCE_HOST') + ENV.delete('KUBE_HOSTNAME') + end end From be44b3244a8b6af1a348642cb5300fc2fe5b47d1 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Wed, 3 Jul 2019 21:18:06 -0400 Subject: [PATCH 46/49] Review fixes --- lib/semian.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/semian.rb b/lib/semian.rb index c59f636a..f10d17b1 100644 --- a/lib/semian.rb +++ b/lib/semian.rb @@ -298,7 +298,7 @@ def require_keys!(required = [], **options) def hostname v = %w(KUBE_HOSTNAME KUBE_HOST_NAME KUBE_NODENAME KUBE_NODE_NAME NODENAME NODE_NAME HOSTNAME HOST_NAME) - var = v.filter { |x| ENV.include?(x) }.first + var = v.find { |x| ENV.include?(x) } ENV[var] if var end From d3f252c6e5361483684b2373f7510fd141863766 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Wed, 3 Jul 2019 22:13:54 -0400 Subject: [PATCH 47/49] Review fixes --- ext/semian/simple_integer.c | 2 +- ext/semian/sliding_window.c | 24 +++++++++----------- ext/semian/sysv_semaphores.c | 2 +- ext/semian/sysv_shared_memory.c | 36 ++++++++++-------------------- ext/semian/sysv_shared_memory.h | 6 ++--- ext/semian/test/Makefile | 18 +++++++++++++++ ext/semian/test/types_test.c | 36 ++++++++++++++++++++++++++++++ ext/semian/types.h | 2 +- test/simple_sliding_window_test.rb | 15 ++++++++++--- 9 files changed, 94 insertions(+), 47 deletions(-) create mode 100644 ext/semian/test/Makefile create mode 100644 ext/semian/test/types_test.c diff --git a/ext/semian/simple_integer.c b/ext/semian/simple_integer.c index 84a68b9c..48cf4100 100644 --- a/ext/semian/simple_integer.c +++ b/ext/semian/simple_integer.c @@ -97,7 +97,7 @@ semian_simple_integer_increment(int argc, VALUE *argv, VALUE self) rb_scan_args(argc, argv, "01", &val); int value = check_increment_arg(val); - if (perform_semop(res->sem_id, 0, value, SEM_UNDO, NULL) == -1) { + if (perform_semop(res->sem_id, 0, value, 0, NULL) == -1) { rb_raise(eInternal, "error incrementing simple integer, errno: %d (%s)", errno, strerror(errno)); } diff --git a/ext/semian/sliding_window.c b/ext/semian/sliding_window.c index 17564bbc..db211a30 100644 --- a/ext/semian/sliding_window.c +++ b/ext/semian/sliding_window.c @@ -28,14 +28,6 @@ static const rb_data_type_t semian_simple_sliding_window_type = { .flags = RUBY_TYPED_FREE_IMMEDIATELY, }; -static void init_fn(void* ptr) -{ - semian_simple_sliding_window_shared_t* res = (semian_simple_sliding_window_shared_t*)ptr; - res->max_size = 0; - res->length = 0; - res->start = 0; -} - static int check_max_size_arg(VALUE max_size) { @@ -93,7 +85,10 @@ check_scale_factor_arg(VALUE scale_factor) static VALUE grow_window(int sem_id, semian_simple_sliding_window_shared_t* window, int new_max_size) { - if (new_max_size > SLIDING_WINDOW_MAX_SIZE) return Qnil; + if (new_max_size > SLIDING_WINDOW_MAX_SIZE) { + sem_meta_unlock(sem_id); + rb_raise(rb_eArgError, "Cannot grow window to %d (MAX_SIZE=%d)", new_max_size, SLIDING_WINDOW_MAX_SIZE); + } int end = window->max_size ? (window->start + window->length) % window->max_size : 0; dprintf("Growing window - sem_id:%d start:%d end:%d length:%d max_size:%d new_max_size:%d", sem_id, window->start, end, window->length, window->max_size, new_max_size); @@ -127,7 +122,10 @@ static void swap(int *a, int *b) { static VALUE shrink_window(int sem_id, semian_simple_sliding_window_shared_t* window, int new_max_size) { - if (new_max_size > SLIDING_WINDOW_MAX_SIZE) return Qnil; + if (new_max_size > SLIDING_WINDOW_MAX_SIZE) { + sem_meta_unlock(sem_id); + rb_raise(rb_eArgError, "Cannot shrink window to %d (MAX_SIZE=%d)", new_max_size, SLIDING_WINDOW_MAX_SIZE); + } int new_length = (new_max_size > window->length) ? window->length : new_max_size; @@ -256,7 +254,7 @@ semian_simple_sliding_window_initialize(VALUE self, VALUE name, VALUE max_size, dprintf("Initializing simple sliding window '%s' (key: %lu)", buffer, res->key); res->sem_id = initialize_single_semaphore(res->key, SEM_DEFAULT_PERMISSIONS, 1); - res->shmem = get_or_create_shared_memory(res->key, init_fn); + res->shmem = get_or_create_shared_memory(res->key); res->error_threshold = check_max_size_arg(max_size); res->scale_factor = check_scale_factor_arg(scale_factor); @@ -396,8 +394,8 @@ semian_simple_sliding_window_clear(VALUE self) return self; } +#if 0 // Handy for debugging the sliding window, but too noisy for regular debugging. -/* static void dprint_window(semian_simple_sliding_window_shared_t *window) { dprintf("---"); @@ -407,7 +405,7 @@ static void dprint_window(semian_simple_sliding_window_shared_t *window) } dprintf("---"); } -*/ +#endif VALUE semian_simple_sliding_window_reject(VALUE self) diff --git a/ext/semian/sysv_semaphores.c b/ext/semian/sysv_semaphores.c index b63c7367..7c0b0c52 100644 --- a/ext/semian/sysv_semaphores.c +++ b/ext/semian/sysv_semaphores.c @@ -214,7 +214,7 @@ wait_for_new_semaphore_set(uint64_t key, long permissions) union semun sem_opts; sem_opts.buf = &sem_ds; - int sem_id = semget(key, 1, permissions); + int sem_id = semget((key_t)key, 1, permissions); if (sem_id == -1){ raise_semian_syscall_error("semget()", errno); } diff --git a/ext/semian/sysv_shared_memory.c b/ext/semian/sysv_shared_memory.c index be1e4b7d..92aeffa1 100644 --- a/ext/semian/sysv_shared_memory.c +++ b/ext/semian/sysv_shared_memory.c @@ -2,26 +2,8 @@ #include "util.h" -#define TIMEOUT_MS (5 * 1e6) -#define WAIT_MS (10) -#define RETRIES (TIMEOUT_MS / WAIT_MS) - void* -wait_for_shared_memory(uint64_t key) -{ - for (int i = 0; i < RETRIES; ++i) { - int shmid = shmget(key, SHM_DEFAULT_SIZE, SHM_DEFAULT_PERMISSIONS); - if (shmid != -1) { - return shmat(shmid, NULL, 0); - } - usleep(WAIT_MS); - } - - rb_raise(rb_eArgError, "could not get shared memory"); -} - -void* -get_or_create_shared_memory(uint64_t key, shared_memory_init_fn fn) +get_or_create_shared_memory(uint64_t key) { void* shmem = NULL; if (!key) return NULL; @@ -29,18 +11,24 @@ get_or_create_shared_memory(uint64_t key, shared_memory_init_fn fn) dprintf("Creating shared memory (key: %lu)", key); int shmid = shmget(key, SHM_DEFAULT_SIZE, IPC_CREAT | IPC_EXCL | SHM_DEFAULT_PERMISSIONS); if (shmid != -1) { - dprintf("Created shared memory (key: %lu)", key); - + dprintf("Created shared memory (key:%lu sem_id:%d)", key, shmid); shmem = shmat(shmid, NULL, 0); if (shmem == (void*)-1) { rb_raise(rb_eArgError, "could not get shared memory (%s)", strerror(errno)); } - if (fn) fn(shmem); - shmctl(key, IPC_RMID, NULL); } else { - shmem = wait_for_shared_memory(key); + shmid = shmget(key, SHM_DEFAULT_SIZE, SHM_DEFAULT_PERMISSIONS); + if (shmid == -1) { + rb_raise(rb_eArgError, "could not get shared memory (%s)", strerror(errno)); + } + + dprintf("Got shared memory (key:%lu sem_id:%d)", key, shmid); + shmem = shmat(shmid, NULL, 0); + if (shmem == (void*)-1) { + rb_raise(rb_eArgError, "could not get shared memory (%s)", strerror(errno)); + } } return shmem; diff --git a/ext/semian/sysv_shared_memory.h b/ext/semian/sysv_shared_memory.h index ee84560c..9528dd46 100644 --- a/ext/semian/sysv_shared_memory.h +++ b/ext/semian/sysv_shared_memory.h @@ -8,12 +8,10 @@ // Default permissions for shared memory #define SHM_DEFAULT_PERMISSIONS 0660 -#define SHM_DEFAULT_SIZE 1024 - -typedef void (*shared_memory_init_fn)(void*); +#define SHM_DEFAULT_SIZE 4096 void* -get_or_create_shared_memory(uint64_t key, shared_memory_init_fn fn); +get_or_create_shared_memory(uint64_t key); void free_shared_memory(void* key); diff --git a/ext/semian/test/Makefile b/ext/semian/test/Makefile new file mode 100644 index 00000000..ad6496c9 --- /dev/null +++ b/ext/semian/test/Makefile @@ -0,0 +1,18 @@ +# Makefile + +CC := gcc -std=gnu99 -I. -I.. +TESTS := types_test + +all : test + +test : types_test + +types_test : types_test.c + $(CC) -o $@ $< + ./$@ + +clean : + rm -f *.o + rm -f $(TESTS) + +.PHONY : all test types_test clean diff --git a/ext/semian/test/types_test.c b/ext/semian/test/types_test.c new file mode 100644 index 00000000..61085574 --- /dev/null +++ b/ext/semian/test/types_test.c @@ -0,0 +1,36 @@ +#include +#include + +#include "types.h" + +void assert_equal_int(int actual, int expected, const char *message) +{ + if (actual != expected) { + fprintf(stderr, "Error: got %d, expected %d (%s)\n", actual, expected, message); + exit(EXIT_FAILURE); + } +} + +void assert_le_int(int actual, int expected, const char *message) +{ + if (actual > expected) { + fprintf(stderr, "Error: got %d, which is greater than %d (%s)\n", actual, expected, message); + exit(EXIT_FAILURE); + } +} + +void test_sliding_window() +{ + semian_simple_sliding_window_shared_t window; + assert_le_int(sizeof(window), 4096, "window size is greater than a page"); + assert_equal_int(sizeof(window.data), SLIDING_WINDOW_MAX_SIZE * sizeof(int), "window data size"); +} + +int main(int argc, char **argv) +{ + printf("Info: Running test\n"); + + test_sliding_window(); + + return EXIT_SUCCESS; +} diff --git a/ext/semian/types.h b/ext/semian/types.h index b6a10a72..81d31d5d 100644 --- a/ext/semian/types.h +++ b/ext/semian/types.h @@ -10,7 +10,7 @@ For custom type definitions specific to semian #include #include -#define SLIDING_WINDOW_MAX_SIZE 4096 +#define SLIDING_WINDOW_MAX_SIZE 1000 // For sysV semop syscals // see man semop diff --git a/test/simple_sliding_window_test.rb b/test/simple_sliding_window_test.rb index 56813bcb..ffafd599 100644 --- a/test/simple_sliding_window_test.rb +++ b/test/simple_sliding_window_test.rb @@ -218,9 +218,18 @@ def test_scale_factor end end - def test_max_size - 8192.times do |i| - @sliding_window << i + def test_huge_sliding_window + id = Time.now.strftime('%H:%M:%S.%N') + window = ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 1000) + 4000.times do |i| + window << i + end + end + + def test_huge_sliding_window_fails + id = Time.now.strftime('%H:%M:%S.%N') + assert_raises ArgumentError do + ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 1001) end end From ea78022693166071c0e2aed7d1be63086d1f7c33 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Thu, 4 Jul 2019 11:51:45 -0400 Subject: [PATCH 48/49] Review comments --- lib/semian.rb | 4 ++-- test/semian_test.rb | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/semian.rb b/lib/semian.rb index f10d17b1..ebeef4ee 100644 --- a/lib/semian.rb +++ b/lib/semian.rb @@ -302,13 +302,13 @@ def hostname ENV[var] if var end -def force_host_circuits +def force_host_circuits? return false unless ENV.include?('SEMIAN_CIRCUIT_BREAKER_FORCE_HOST') ENV['SEMIAN_CIRCUIT_BREAKER_FORCE_HOST'].split(',').include?(hostname) end if Semian.semaphores_enabled? - ENV['SEMIAN_CIRCUIT_BREAKER_IMPL'] = 'host' if force_host_circuits + ENV['SEMIAN_CIRCUIT_BREAKER_IMPL'] = 'host' if force_host_circuits? require 'semian/semian' else Semian::MAX_TICKETS = 0 diff --git a/test/semian_test.rb b/test/semian_test.rb index 568ffcb2..db504b6f 100644 --- a/test/semian_test.rb +++ b/test/semian_test.rb @@ -93,11 +93,11 @@ def test_disabled_via_semian_wide_env_var end def test_force_host_circuits - refute force_host_circuits + refute force_host_circuits? ENV['SEMIAN_CIRCUIT_BREAKER_FORCE_HOST'] = 'machine-1,machine-2,machine-3' - refute force_host_circuits + refute force_host_circuits? ENV['KUBE_HOSTNAME'] = 'machine-2' - assert force_host_circuits + assert force_host_circuits? ensure ENV.delete('SEMIAN_CIRCUIT_BREAKER_FORCE_HOST') ENV.delete('KUBE_HOSTNAME') From 286b6c12af6c9e44d05e499592434dd9862c3f09 Mon Sep 17 00:00:00 2001 From: Michael Kipper Date: Mon, 8 Jul 2019 12:50:21 -0400 Subject: [PATCH 49/49] Skip test --- test/simple_sliding_window_test.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/simple_sliding_window_test.rb b/test/simple_sliding_window_test.rb index ffafd599..a96efd55 100644 --- a/test/simple_sliding_window_test.rb +++ b/test/simple_sliding_window_test.rb @@ -227,6 +227,7 @@ def test_huge_sliding_window end def test_huge_sliding_window_fails + skip unless ENV['SEMIAN_CIRCUIT_BREAKER_IMPL'] == 'host' id = Time.now.strftime('%H:%M:%S.%N') assert_raises ArgumentError do ::Semian::ThreadSafe::SlidingWindow.new(id, max_size: 1001)