From 49f5a29c9c5b4d8de3517ee49cb7d1ec790e0d50 Mon Sep 17 00:00:00 2001 From: Daijiro Fukuda Date: Thu, 24 Mar 2022 12:50:34 +0900 Subject: [PATCH] Add scenario test of RetryState RetryStatement behavior is complicated, so scenario tests may be useful. Signed-off-by: Daijiro Fukuda --- test/plugin_helper/test_retry_state.rb | 570 +++++++++++++++++++++++++ 1 file changed, 570 insertions(+) diff --git a/test/plugin_helper/test_retry_state.rb b/test/plugin_helper/test_retry_state.rb index a6cd7300dc..824151c3a8 100644 --- a/test/plugin_helper/test_retry_state.rb +++ b/test/plugin_helper/test_retry_state.rb @@ -18,6 +18,21 @@ class Dummy < Fluent::Plugin::TestBase helpers :retry_state end + class RetryRecord + attr_reader :retry_count, :elapsed_sec, :is_secondary + def initialize(retry_count, elapsed_sec, is_secondary) + @retry_count = retry_count # This is Nth retryment + @elapsed_sec = elapsed_sec + @is_secondary = is_secondary + end + + def ==(obj) + @retry_count == obj.retry_count && + @elapsed_sec == obj.elapsed_sec && + @is_secondary == obj.is_secondary + end + end + setup do @d = Dummy.new end @@ -436,4 +451,559 @@ class Dummy < Fluent::Plugin::TestBase end end end + + sub_test_case "ExponentialBackOff_ScenarioTests" do + data("Simple timeout", { + timeout: 100, max_steps: nil, max_interval: nil, use_sec: false, sec_thres: 0.8, wait: 1, backoff_base: 2, + expected: [ + RetryRecord.new(1, 1, false), + RetryRecord.new(2, 3, false), + RetryRecord.new(3, 7, false), + RetryRecord.new(4, 15, false), + RetryRecord.new(5, 31, false), + RetryRecord.new(6, 63, false), + RetryRecord.new(7, 100, false), + ], + }) + data("Simple timeout with secondary", { + timeout: 100, max_steps: nil, max_interval: nil, use_sec: true, sec_thres: 0.8, wait: 1, backoff_base: 2, + expected: [ + RetryRecord.new(1, 1, false), + RetryRecord.new(2, 3, false), + RetryRecord.new(3, 7, false), + RetryRecord.new(4, 15, false), + RetryRecord.new(5, 31, false), + RetryRecord.new(6, 63, false), + RetryRecord.new(7, 80, true), + RetryRecord.new(8, 81, true), + RetryRecord.new(9, 83, true), + RetryRecord.new(10, 87, true), + RetryRecord.new(11, 95, true), + RetryRecord.new(12, 100, true), + ], + }) + data("Simple timeout with custom wait and backoff_base", { + timeout: 1000, max_steps: nil, max_interval: nil, use_sec: false, sec_thres: 0.8, wait: 2, backoff_base: 3, + expected: [ + RetryRecord.new(1, 2, false), + RetryRecord.new(2, 8, false), + RetryRecord.new(3, 26, false), + RetryRecord.new(4, 80, false), + RetryRecord.new(5, 242, false), + RetryRecord.new(6, 728, false), + RetryRecord.new(7, 1000, false), + ], + }) + data("Simple timeout with custom wait and backoff_base and secondary", { + timeout: 1000, max_steps: nil, max_interval: nil, use_sec: true, sec_thres: 0.8, wait: 2, backoff_base: 3, + expected: [ + RetryRecord.new(1, 2, false), + RetryRecord.new(2, 8, false), + RetryRecord.new(3, 26, false), + RetryRecord.new(4, 80, false), + RetryRecord.new(5, 242, false), + RetryRecord.new(6, 728, false), + RetryRecord.new(7, 800, true), + RetryRecord.new(8, 802, true), + RetryRecord.new(9, 808, true), + RetryRecord.new(10, 826, true), + RetryRecord.new(11, 880, true), + RetryRecord.new(12, 1000, true), + ], + }) + data("Default timeout", { + timeout: 72*3600, max_steps: nil, max_interval: nil, use_sec: false, sec_thres: 0.8, wait: 1, backoff_base: 2, + expected: [ + RetryRecord.new(1, 1, false), + RetryRecord.new(2, 3, false), + RetryRecord.new(3, 7, false), + RetryRecord.new(4, 15, false), + RetryRecord.new(5, 31, false), + RetryRecord.new(6, 63, false), + RetryRecord.new(7, 127, false), + RetryRecord.new(8, 255, false), + RetryRecord.new(9, 511, false), + RetryRecord.new(10, 1023, false), + RetryRecord.new(11, 2047, false), + RetryRecord.new(12, 4095, false), + RetryRecord.new(13, 8191, false), + RetryRecord.new(14, 16383, false), + RetryRecord.new(15, 32767, false), + RetryRecord.new(16, 65535, false), + RetryRecord.new(17, 131071, false), + RetryRecord.new(18, 259200, false), + ], + }) + data("Default timeout with secondary", { + timeout: 72*3600, max_steps: nil, max_interval: nil, use_sec: true, sec_thres: 0.8, wait: 1, backoff_base: 2, + expected: [ + RetryRecord.new(1, 1, false), + RetryRecord.new(2, 3, false), + RetryRecord.new(3, 7, false), + RetryRecord.new(4, 15, false), + RetryRecord.new(5, 31, false), + RetryRecord.new(6, 63, false), + RetryRecord.new(7, 127, false), + RetryRecord.new(8, 255, false), + RetryRecord.new(9, 511, false), + RetryRecord.new(10, 1023, false), + RetryRecord.new(11, 2047, false), + RetryRecord.new(12, 4095, false), + RetryRecord.new(13, 8191, false), + RetryRecord.new(14, 16383, false), + RetryRecord.new(15, 32767, false), + RetryRecord.new(16, 65535, false), + RetryRecord.new(17, 131071, false), + RetryRecord.new(18, 207360, true), + RetryRecord.new(19, 207361, true), + RetryRecord.new(20, 207363, true), + RetryRecord.new(21, 207367, true), + RetryRecord.new(22, 207375, true), + RetryRecord.new(23, 207391, true), + RetryRecord.new(24, 207423, true), + RetryRecord.new(25, 207487, true), + RetryRecord.new(26, 207615, true), + RetryRecord.new(27, 207871, true), + RetryRecord.new(28, 208383, true), + RetryRecord.new(29, 209407, true), + RetryRecord.new(30, 211455, true), + RetryRecord.new(31, 215551, true), + RetryRecord.new(32, 223743, true), + RetryRecord.new(33, 240127, true), + RetryRecord.new(34, 259200, true), + ], + }) + data("Default timeout with secondary and custom threshold", { + timeout: 72*3600, max_steps: nil, max_interval: nil, use_sec: true, sec_thres: 0.5, wait: 1, backoff_base: 2, + expected: [ + RetryRecord.new(1, 1, false), + RetryRecord.new(2, 3, false), + RetryRecord.new(3, 7, false), + RetryRecord.new(4, 15, false), + RetryRecord.new(5, 31, false), + RetryRecord.new(6, 63, false), + RetryRecord.new(7, 127, false), + RetryRecord.new(8, 255, false), + RetryRecord.new(9, 511, false), + RetryRecord.new(10, 1023, false), + RetryRecord.new(11, 2047, false), + RetryRecord.new(12, 4095, false), + RetryRecord.new(13, 8191, false), + RetryRecord.new(14, 16383, false), + RetryRecord.new(15, 32767, false), + RetryRecord.new(16, 65535, false), + RetryRecord.new(17, 129600, true), + RetryRecord.new(18, 129601, true), + RetryRecord.new(19, 129603, true), + RetryRecord.new(20, 129607, true), + RetryRecord.new(21, 129615, true), + RetryRecord.new(22, 129631, true), + RetryRecord.new(23, 129663, true), + RetryRecord.new(24, 129727, true), + RetryRecord.new(25, 129855, true), + RetryRecord.new(26, 130111, true), + RetryRecord.new(27, 130623, true), + RetryRecord.new(28, 131647, true), + RetryRecord.new(29, 133695, true), + RetryRecord.new(30, 137791, true), + RetryRecord.new(31, 145983, true), + RetryRecord.new(32, 162367, true), + RetryRecord.new(33, 195135, true), + RetryRecord.new(34, 259200, true), + ], + }) + data("Simple max_steps", { + timeout: 72*3600, max_steps: 10, max_interval: nil, use_sec: false, sec_thres: 0.8, wait: 1, backoff_base: 2, + expected: [ + RetryRecord.new(1, 1, false), + RetryRecord.new(2, 3, false), + RetryRecord.new(3, 7, false), + RetryRecord.new(4, 15, false), + RetryRecord.new(5, 31, false), + RetryRecord.new(6, 63, false), + RetryRecord.new(7, 127, false), + RetryRecord.new(8, 255, false), + RetryRecord.new(9, 511, false), + RetryRecord.new(10, 1023, false), + ], + }) + data("Simple max_steps with secondary", { + timeout: 72*3600, max_steps: 10, max_interval: nil, use_sec: true, sec_thres: 0.8, wait: 1, backoff_base: 2, + expected: [ + RetryRecord.new(1, 1, false), + RetryRecord.new(2, 3, false), + RetryRecord.new(3, 7, false), + RetryRecord.new(4, 15, false), + RetryRecord.new(5, 31, false), + RetryRecord.new(6, 63, false), + RetryRecord.new(7, 127, false), + RetryRecord.new(8, 255, false), + RetryRecord.new(9, 511, false), + RetryRecord.new(10, 818, true), + ], + }) + data("Simple interval", { + timeout: 72*3600, max_steps: nil, max_interval: 3600, use_sec: false, sec_thres: 0.8, wait: 1, backoff_base: 2, + expected: [ + RetryRecord.new(1, 1, false), + RetryRecord.new(2, 3, false), + RetryRecord.new(3, 7, false), + RetryRecord.new(4, 15, false), + RetryRecord.new(5, 31, false), + RetryRecord.new(6, 63, false), + RetryRecord.new(7, 127, false), + RetryRecord.new(8, 255, false), + RetryRecord.new(9, 511, false), + RetryRecord.new(10, 1023, false), + RetryRecord.new(11, 2047, false), + RetryRecord.new(12, 4095, false), + RetryRecord.new(13, 7695, false), + RetryRecord.new(14, 11295, false), + RetryRecord.new(15, 14895, false), + RetryRecord.new(16, 18495, false), + RetryRecord.new(17, 22095, false), + RetryRecord.new(18, 25695, false), + RetryRecord.new(19, 29295, false), + RetryRecord.new(20, 32895, false), + RetryRecord.new(21, 36495, false), + RetryRecord.new(22, 40095, false), + RetryRecord.new(23, 43695, false), + RetryRecord.new(24, 47295, false), + RetryRecord.new(25, 50895, false), + RetryRecord.new(26, 54495, false), + RetryRecord.new(27, 58095, false), + RetryRecord.new(28, 61695, false), + RetryRecord.new(29, 65295, false), + RetryRecord.new(30, 68895, false), + RetryRecord.new(31, 72495, false), + RetryRecord.new(32, 76095, false), + RetryRecord.new(33, 79695, false), + RetryRecord.new(34, 83295, false), + RetryRecord.new(35, 86895, false), + RetryRecord.new(36, 90495, false), + RetryRecord.new(37, 94095, false), + RetryRecord.new(38, 97695, false), + RetryRecord.new(39, 101295, false), + RetryRecord.new(40, 104895, false), + RetryRecord.new(41, 108495, false), + RetryRecord.new(42, 112095, false), + RetryRecord.new(43, 115695, false), + RetryRecord.new(44, 119295, false), + RetryRecord.new(45, 122895, false), + RetryRecord.new(46, 126495, false), + RetryRecord.new(47, 130095, false), + RetryRecord.new(48, 133695, false), + RetryRecord.new(49, 137295, false), + RetryRecord.new(50, 140895, false), + RetryRecord.new(51, 144495, false), + RetryRecord.new(52, 148095, false), + RetryRecord.new(53, 151695, false), + RetryRecord.new(54, 155295, false), + RetryRecord.new(55, 158895, false), + RetryRecord.new(56, 162495, false), + RetryRecord.new(57, 166095, false), + RetryRecord.new(58, 169695, false), + RetryRecord.new(59, 173295, false), + RetryRecord.new(60, 176895, false), + RetryRecord.new(61, 180495, false), + RetryRecord.new(62, 184095, false), + RetryRecord.new(63, 187695, false), + RetryRecord.new(64, 191295, false), + RetryRecord.new(65, 194895, false), + RetryRecord.new(66, 198495, false), + RetryRecord.new(67, 202095, false), + RetryRecord.new(68, 205695, false), + RetryRecord.new(69, 209295, false), + RetryRecord.new(70, 212895, false), + RetryRecord.new(71, 216495, false), + RetryRecord.new(72, 220095, false), + RetryRecord.new(73, 223695, false), + RetryRecord.new(74, 227295, false), + RetryRecord.new(75, 230895, false), + RetryRecord.new(76, 234495, false), + RetryRecord.new(77, 238095, false), + RetryRecord.new(78, 241695, false), + RetryRecord.new(79, 245295, false), + RetryRecord.new(80, 248895, false), + RetryRecord.new(81, 252495, false), + RetryRecord.new(82, 256095, false), + RetryRecord.new(83, 259200, false), + ], + }) + data("Simple interval with secondary", { + timeout: 72*3600, max_steps: nil, max_interval: 3600, use_sec: true, sec_thres: 0.8, wait: 1, backoff_base: 2, + expected: [ + RetryRecord.new(1, 1, false), + RetryRecord.new(2, 3, false), + RetryRecord.new(3, 7, false), + RetryRecord.new(4, 15, false), + RetryRecord.new(5, 31, false), + RetryRecord.new(6, 63, false), + RetryRecord.new(7, 127, false), + RetryRecord.new(8, 255, false), + RetryRecord.new(9, 511, false), + RetryRecord.new(10, 1023, false), + RetryRecord.new(11, 2047, false), + RetryRecord.new(12, 4095, false), + RetryRecord.new(13, 7695, false), + RetryRecord.new(14, 11295, false), + RetryRecord.new(15, 14895, false), + RetryRecord.new(16, 18495, false), + RetryRecord.new(17, 22095, false), + RetryRecord.new(18, 25695, false), + RetryRecord.new(19, 29295, false), + RetryRecord.new(20, 32895, false), + RetryRecord.new(21, 36495, false), + RetryRecord.new(22, 40095, false), + RetryRecord.new(23, 43695, false), + RetryRecord.new(24, 47295, false), + RetryRecord.new(25, 50895, false), + RetryRecord.new(26, 54495, false), + RetryRecord.new(27, 58095, false), + RetryRecord.new(28, 61695, false), + RetryRecord.new(29, 65295, false), + RetryRecord.new(30, 68895, false), + RetryRecord.new(31, 72495, false), + RetryRecord.new(32, 76095, false), + RetryRecord.new(33, 79695, false), + RetryRecord.new(34, 83295, false), + RetryRecord.new(35, 86895, false), + RetryRecord.new(36, 90495, false), + RetryRecord.new(37, 94095, false), + RetryRecord.new(38, 97695, false), + RetryRecord.new(39, 101295, false), + RetryRecord.new(40, 104895, false), + RetryRecord.new(41, 108495, false), + RetryRecord.new(42, 112095, false), + RetryRecord.new(43, 115695, false), + RetryRecord.new(44, 119295, false), + RetryRecord.new(45, 122895, false), + RetryRecord.new(46, 126495, false), + RetryRecord.new(47, 130095, false), + RetryRecord.new(48, 133695, false), + RetryRecord.new(49, 137295, false), + RetryRecord.new(50, 140895, false), + RetryRecord.new(51, 144495, false), + RetryRecord.new(52, 148095, false), + RetryRecord.new(53, 151695, false), + RetryRecord.new(54, 155295, false), + RetryRecord.new(55, 158895, false), + RetryRecord.new(56, 162495, false), + RetryRecord.new(57, 166095, false), + RetryRecord.new(58, 169695, false), + RetryRecord.new(59, 173295, false), + RetryRecord.new(60, 176895, false), + RetryRecord.new(61, 180495, false), + RetryRecord.new(62, 184095, false), + RetryRecord.new(63, 187695, false), + RetryRecord.new(64, 191295, false), + RetryRecord.new(65, 194895, false), + RetryRecord.new(66, 198495, false), + RetryRecord.new(67, 202095, false), + RetryRecord.new(68, 205695, false), + RetryRecord.new(69, 207360, true), + RetryRecord.new(70, 207361, true), + RetryRecord.new(71, 207363, true), + RetryRecord.new(72, 207367, true), + RetryRecord.new(73, 207375, true), + RetryRecord.new(74, 207391, true), + RetryRecord.new(75, 207423, true), + RetryRecord.new(76, 207487, true), + RetryRecord.new(77, 207615, true), + RetryRecord.new(78, 207871, true), + RetryRecord.new(79, 208383, true), + RetryRecord.new(80, 209407, true), + RetryRecord.new(81, 211455, true), + RetryRecord.new(82, 215055, true), + RetryRecord.new(83, 218655, true), + RetryRecord.new(84, 222255, true), + RetryRecord.new(85, 225855, true), + RetryRecord.new(86, 229455, true), + RetryRecord.new(87, 233055, true), + RetryRecord.new(88, 236655, true), + RetryRecord.new(89, 240255, true), + RetryRecord.new(90, 243855, true), + RetryRecord.new(91, 247455, true), + RetryRecord.new(92, 251055, true), + RetryRecord.new(93, 254655, true), + RetryRecord.new(94, 258255, true), + RetryRecord.new(95, 259200, true), + ], + }) + data("Max_steps and max_interval", { + timeout: 72*3600, max_steps: 30, max_interval: 3600, use_sec: false, sec_thres: 0.8, wait: 1, backoff_base: 2, + expected: [ + RetryRecord.new(1, 1, false), + RetryRecord.new(2, 3, false), + RetryRecord.new(3, 7, false), + RetryRecord.new(4, 15, false), + RetryRecord.new(5, 31, false), + RetryRecord.new(6, 63, false), + RetryRecord.new(7, 127, false), + RetryRecord.new(8, 255, false), + RetryRecord.new(9, 511, false), + RetryRecord.new(10, 1023, false), + RetryRecord.new(11, 2047, false), + RetryRecord.new(12, 4095, false), + RetryRecord.new(13, 7695, false), + RetryRecord.new(14, 11295, false), + RetryRecord.new(15, 14895, false), + RetryRecord.new(16, 18495, false), + RetryRecord.new(17, 22095, false), + RetryRecord.new(18, 25695, false), + RetryRecord.new(19, 29295, false), + RetryRecord.new(20, 32895, false), + RetryRecord.new(21, 36495, false), + RetryRecord.new(22, 40095, false), + RetryRecord.new(23, 43695, false), + RetryRecord.new(24, 47295, false), + RetryRecord.new(25, 50895, false), + RetryRecord.new(26, 54495, false), + RetryRecord.new(27, 58095, false), + RetryRecord.new(28, 61695, false), + RetryRecord.new(29, 65295, false), + RetryRecord.new(30, 68895, false), + ], + }) + data("Max_steps and max_interval with secondary", { + timeout: 72*3600, max_steps: 30, max_interval: 3600, use_sec: true, sec_thres: 0.8, wait: 1, backoff_base: 2, + expected: [ + RetryRecord.new(1, 1, false), + RetryRecord.new(2, 3, false), + RetryRecord.new(3, 7, false), + RetryRecord.new(4, 15, false), + RetryRecord.new(5, 31, false), + RetryRecord.new(6, 63, false), + RetryRecord.new(7, 127, false), + RetryRecord.new(8, 255, false), + RetryRecord.new(9, 511, false), + RetryRecord.new(10, 1023, false), + RetryRecord.new(11, 2047, false), + RetryRecord.new(12, 4095, false), + RetryRecord.new(13, 7695, false), + RetryRecord.new(14, 11295, false), + RetryRecord.new(15, 14895, false), + RetryRecord.new(16, 18495, false), + RetryRecord.new(17, 22095, false), + RetryRecord.new(18, 25695, false), + RetryRecord.new(19, 29295, false), + RetryRecord.new(20, 32895, false), + RetryRecord.new(21, 36495, false), + RetryRecord.new(22, 40095, false), + RetryRecord.new(23, 43695, false), + RetryRecord.new(24, 47295, false), + RetryRecord.new(25, 50895, false), + RetryRecord.new(26, 54495, false), + RetryRecord.new(27, 55116, true), + RetryRecord.new(28, 55117, true), + RetryRecord.new(29, 55119, true), + RetryRecord.new(30, 55123, true), + ], + }) + data("Max_steps and max_interval with timeout", { + timeout: 10000, max_steps: 30, max_interval: 1000, use_sec: false, sec_thres: 0.8, wait: 1, backoff_base: 2, + expected: [ + RetryRecord.new(1, 1, false), + RetryRecord.new(2, 3, false), + RetryRecord.new(3, 7, false), + RetryRecord.new(4, 15, false), + RetryRecord.new(5, 31, false), + RetryRecord.new(6, 63, false), + RetryRecord.new(7, 127, false), + RetryRecord.new(8, 255, false), + RetryRecord.new(9, 511, false), + RetryRecord.new(10, 1023, false), + RetryRecord.new(11, 2023, false), + RetryRecord.new(12, 3023, false), + RetryRecord.new(13, 4023, false), + RetryRecord.new(14, 5023, false), + RetryRecord.new(15, 6023, false), + RetryRecord.new(16, 7023, false), + RetryRecord.new(17, 8023, false), + RetryRecord.new(18, 9023, false), + RetryRecord.new(19, 10000, false), + ], + }) + data("Max_steps and max_interval with timeout and secondary", { + timeout: 10000, max_steps: 30, max_interval: 1000, use_sec: true, sec_thres: 0.8, wait: 1, backoff_base: 2, + expected: [ + RetryRecord.new(1, 1, false), + RetryRecord.new(2, 3, false), + RetryRecord.new(3, 7, false), + RetryRecord.new(4, 15, false), + RetryRecord.new(5, 31, false), + RetryRecord.new(6, 63, false), + RetryRecord.new(7, 127, false), + RetryRecord.new(8, 255, false), + RetryRecord.new(9, 511, false), + RetryRecord.new(10, 1023, false), + RetryRecord.new(11, 2023, false), + RetryRecord.new(12, 3023, false), + RetryRecord.new(13, 4023, false), + RetryRecord.new(14, 5023, false), + RetryRecord.new(15, 6023, false), + RetryRecord.new(16, 7023, false), + RetryRecord.new(17, 8000, true), + RetryRecord.new(18, 8001, true), + RetryRecord.new(19, 8003, true), + RetryRecord.new(20, 8007, true), + RetryRecord.new(21, 8015, true), + RetryRecord.new(22, 8031, true), + RetryRecord.new(23, 8063, true), + RetryRecord.new(24, 8127, true), + RetryRecord.new(25, 8255, true), + RetryRecord.new(26, 8511, true), + RetryRecord.new(27, 9023, true), + RetryRecord.new(28, 10000, true), + ], + }) + test "exponential backoff with senario" do |data| + print_for_debug = false # change this value true if need to see msg always. + trying_count = 1000 # just for avoiding infinite loop + + retry_records = [] + msg = "" + + s = @d.retry_state_create( + :t15, :exponential_backoff, data[:wait], data[:timeout], + max_steps: data[:max_steps], max_interval: data[:max_interval], + secondary: data[:use_sec], secondary_threshold: data[:sec_thres], + backoff_base: data[:backoff_base], randomize: false + ) + override_current_time(s, s.start) + + retry_count = 0 + trying_count.times do + next_elapsed = (s.next_time - s.start).to_i + + msg << "step: #{s.steps}, next: #{next_elapsed}s (#{next_elapsed / 3600}h)\n" + + # Wait until next time to trigger the next retry + override_current_time(s, s.next_time) + + # Retry will be triggered at this point. + retry_count += 1 + rec = RetryRecord.new(retry_count, next_elapsed, s.secondary?) + retry_records.append(rec) + msg << "[#{next_elapsed}s elapsed point] #{retry_count}th-Retry(#{s.secondary? ? "SEC" : "PRI"}) is triggered.\n" + + # Update retry statement + if s.limit? + msg << "--- Reach limit of timeout. ---\n" + break + end + + s.step + + if s.limit_step? + msg << "--- Reach limit of max step. ---\n" + break + end + end + + assert_equal(data[:expected], retry_records, msg) + + print(msg) if print_for_debug + end + end end