-
Notifications
You must be signed in to change notification settings - Fork 0
/
HystrixCircuitBreakerTest.java
834 lines (683 loc) · 30.8 KB
/
HystrixCircuitBreakerTest.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
/**
* Copyright 2015 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.netflix.hystrix;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.util.Random;
import com.hystrix.junit.HystrixRequestContextRule;
import com.netflix.hystrix.exception.HystrixBadRequestException;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import com.netflix.hystrix.HystrixCircuitBreaker.HystrixCircuitBreakerImpl;
import com.netflix.hystrix.strategy.HystrixPlugins;
import com.netflix.hystrix.strategy.executionhook.HystrixCommandExecutionHook;
import rx.Observable;
import rx.Subscription;
/**
* These tests each use a different command key to ensure that running them in parallel doesn't allow the state
* built up during a test to cause others to fail
*/
public class HystrixCircuitBreakerTest {
@Rule
public HystrixRequestContextRule ctx = new HystrixRequestContextRule();
@Before
public void init() {
for (HystrixCommandMetrics metricsInstance: HystrixCommandMetrics.getInstances()) {
metricsInstance.resetStream();
}
HystrixCommandMetrics.reset();
HystrixCircuitBreaker.Factory.reset();
Hystrix.reset();
}
/**
* A simple circuit breaker intended for unit testing of the {@link HystrixCommand} object, NOT production use.
* <p>
* This uses simple logic to 'trip' the circuit after 3 subsequent failures and doesn't recover.
*/
public static class TestCircuitBreaker implements HystrixCircuitBreaker {
final HystrixCommandMetrics metrics;
private boolean forceShortCircuit = false;
public TestCircuitBreaker() {
this.metrics = getMetrics(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter());
forceShortCircuit = false;
}
public TestCircuitBreaker(HystrixCommandKey commandKey) {
this.metrics = getMetrics(commandKey, HystrixCommandPropertiesTest.getUnitTestPropertiesSetter());
forceShortCircuit = false;
}
public TestCircuitBreaker setForceShortCircuit(boolean value) {
this.forceShortCircuit = value;
return this;
}
@Override
public boolean isOpen() {
System.out.println("metrics : " + metrics.getCommandKey().name() + " : " + metrics.getHealthCounts());
if (forceShortCircuit) {
return true;
} else {
return metrics.getHealthCounts().getErrorCount() >= 3;
}
}
@Override
public void markSuccess() {
// we don't need to do anything since we're going to permanently trip the circuit
}
@Override
public void markNonSuccess() {
}
@Override
public boolean attemptExecution() {
return !isOpen();
}
@Override
public boolean allowRequest() {
return !isOpen();
}
}
/**
* Test that if all 'marks' are successes during the test window that it does NOT trip the circuit.
* Test that if all 'marks' are failures during the test window that it trips the circuit.
*/
@Test
public void testTripCircuit() {
String key = "cmd-A";
try {
HystrixCommand<Boolean> cmd1 = new SuccessCommand(key, 1);
HystrixCommand<Boolean> cmd2 = new SuccessCommand(key, 1);
HystrixCommand<Boolean> cmd3 = new SuccessCommand(key, 1);
HystrixCommand<Boolean> cmd4 = new SuccessCommand(key, 1);
HystrixCircuitBreaker cb = cmd1.circuitBreaker;
cmd1.execute();
cmd2.execute();
cmd3.execute();
cmd4.execute();
// this should still allow requests as everything has been successful
Thread.sleep(100);
//assertTrue(cb.allowRequest());
assertFalse(cb.isOpen());
// fail
HystrixCommand<Boolean> cmd5 = new FailureCommand(key, 1);
HystrixCommand<Boolean> cmd6 = new FailureCommand(key, 1);
HystrixCommand<Boolean> cmd7 = new FailureCommand(key, 1);
HystrixCommand<Boolean> cmd8 = new FailureCommand(key, 1);
cmd5.execute();
cmd6.execute();
cmd7.execute();
cmd8.execute();
// everything has failed in the test window so we should return false now
Thread.sleep(100);
assertFalse(cb.allowRequest());
assertTrue(cb.isOpen());
} catch (Exception e) {
e.printStackTrace();
fail("Error occurred: " + e.getMessage());
}
}
/**
* Test that if the % of failures is higher than the threshold that the circuit trips.
*/
@Test
public void testTripCircuitOnFailuresAboveThreshold() {
String key = "cmd-B";
try {
HystrixCommand<Boolean> cmd1 = new SuccessCommand(key, 60);
HystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
assertTrue(cb.allowRequest());
assertFalse(cb.isOpen());
// success with high latency
cmd1.execute();
HystrixCommand<Boolean> cmd2 = new SuccessCommand(key, 1);
cmd2.execute();
HystrixCommand<Boolean> cmd3 = new FailureCommand(key, 1);
cmd3.execute();
HystrixCommand<Boolean> cmd4 = new SuccessCommand(key, 1);
cmd4.execute();
HystrixCommand<Boolean> cmd5 = new FailureCommand(key, 1);
cmd5.execute();
HystrixCommand<Boolean> cmd6 = new SuccessCommand(key, 1);
cmd6.execute();
HystrixCommand<Boolean> cmd7 = new FailureCommand(key, 1);
cmd7.execute();
HystrixCommand<Boolean> cmd8 = new FailureCommand(key, 1);
cmd8.execute();
// this should trip the circuit as the error percentage is above the threshold
Thread.sleep(100);
assertFalse(cb.allowRequest());
assertTrue(cb.isOpen());
} catch (Exception e) {
e.printStackTrace();
fail("Error occurred: " + e.getMessage());
}
}
/**
* Test that if the % of failures is higher than the threshold that the circuit trips.
*/
@Test
public void testCircuitDoesNotTripOnFailuresBelowThreshold() {
String key = "cmd-C";
try {
HystrixCommand<Boolean> cmd1 = new SuccessCommand(key, 60);
HystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
assertTrue(cb.allowRequest());
assertFalse(cb.isOpen());
// success with high latency
cmd1.execute();
HystrixCommand<Boolean> cmd2 = new SuccessCommand(key, 1);
cmd2.execute();
HystrixCommand<Boolean> cmd3 = new FailureCommand(key, 1);
cmd3.execute();
HystrixCommand<Boolean> cmd4 = new SuccessCommand(key, 1);
cmd4.execute();
HystrixCommand<Boolean> cmd5 = new SuccessCommand(key, 1);
cmd5.execute();
HystrixCommand<Boolean> cmd6 = new FailureCommand(key, 1);
cmd6.execute();
HystrixCommand<Boolean> cmd7 = new SuccessCommand(key, 1);
cmd7.execute();
HystrixCommand<Boolean> cmd8 = new FailureCommand(key, 1);
cmd8.execute();
// this should remain closed as the failure threshold is below the percentage limit
Thread.sleep(100);
System.out.println("ReqLog : " + HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString());
System.out.println("Current CircuitBreaker Status : " + cmd1.getMetrics().getHealthCounts());
assertTrue(cb.allowRequest());
assertFalse(cb.isOpen());
} catch (Exception e) {
e.printStackTrace();
fail("Error occurred: " + e.getMessage());
}
}
/**
* Test that if all 'marks' are timeouts that it will trip the circuit.
*/
@Test
public void testTripCircuitOnTimeouts() {
String key = "cmd-D";
try {
HystrixCommand<Boolean> cmd1 = new TimeoutCommand(key);
HystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
assertTrue(cb.allowRequest());
assertFalse(cb.isOpen());
// success with high latency
cmd1.execute();
HystrixCommand<Boolean> cmd2 = new TimeoutCommand(key);
cmd2.execute();
HystrixCommand<Boolean> cmd3 = new TimeoutCommand(key);
cmd3.execute();
HystrixCommand<Boolean> cmd4 = new TimeoutCommand(key);
cmd4.execute();
// everything has been a timeout so we should not allow any requests
Thread.sleep(100);
assertFalse(cb.allowRequest());
assertTrue(cb.isOpen());
} catch (Exception e) {
e.printStackTrace();
fail("Error occurred: " + e.getMessage());
}
}
/**
* Test that if the % of timeouts is higher than the threshold that the circuit trips.
*/
@Test
public void testTripCircuitOnTimeoutsAboveThreshold() {
String key = "cmd-E";
try {
HystrixCommand<Boolean> cmd1 = new SuccessCommand(key, 60);
HystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
assertTrue(cb.allowRequest());
assertFalse(cb.isOpen());
// success with high latency
cmd1.execute();
HystrixCommand<Boolean> cmd2 = new SuccessCommand(key, 1);
cmd2.execute();
HystrixCommand<Boolean> cmd3 = new TimeoutCommand(key);
cmd3.execute();
HystrixCommand<Boolean> cmd4 = new SuccessCommand(key, 1);
cmd4.execute();
HystrixCommand<Boolean> cmd5 = new TimeoutCommand(key);
cmd5.execute();
HystrixCommand<Boolean> cmd6 = new TimeoutCommand(key);
cmd6.execute();
HystrixCommand<Boolean> cmd7 = new SuccessCommand(key, 1);
cmd7.execute();
HystrixCommand<Boolean> cmd8 = new TimeoutCommand(key);
cmd8.execute();
HystrixCommand<Boolean> cmd9 = new TimeoutCommand(key);
cmd9.execute();
// this should trip the circuit as the error percentage is above the threshold
Thread.sleep(100);
assertTrue(cb.isOpen());
} catch (Exception e) {
e.printStackTrace();
fail("Error occurred: " + e.getMessage());
}
}
/**
* Test that on an open circuit that a single attempt will be allowed after a window of time to see if issues are resolved.
*/
@Test
public void testSingleTestOnOpenCircuitAfterTimeWindow() {
String key = "cmd-F";
try {
int sleepWindow = 200;
HystrixCommand<Boolean> cmd1 = new FailureCommand(key, 60);
HystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
assertTrue(cb.allowRequest());
assertFalse(cb.isOpen());
cmd1.execute();
HystrixCommand<Boolean> cmd2 = new FailureCommand(key, 1);
cmd2.execute();
HystrixCommand<Boolean> cmd3 = new FailureCommand(key, 1);
cmd3.execute();
HystrixCommand<Boolean> cmd4 = new FailureCommand(key, 1);
cmd4.execute();
// everything has failed in the test window so we should return false now
Thread.sleep(100);
assertFalse(cb.allowRequest());
assertTrue(cb.isOpen());
// wait for sleepWindow to pass
Thread.sleep(sleepWindow + 50);
// we should now allow 1 request
assertTrue(cb.attemptExecution());
// but the circuit should still be open
assertTrue(cb.isOpen());
// and further requests are still blocked
assertFalse(cb.attemptExecution());
} catch (Exception e) {
e.printStackTrace();
fail("Error occurred: " + e.getMessage());
}
}
/**
* Test that an open circuit is closed after 1 success. This also ensures that the rolling window (containing failures) is cleared after the sleep window
* Otherwise, the next bucket roll would produce another signal to fail unless it is explicitly cleared (via {@link HystrixCommandMetrics#resetStream()}.
*/
@Test
public void testCircuitClosedAfterSuccess() {
String key = "cmd-G";
try {
int sleepWindow = 100;
HystrixCommand<Boolean> cmd1 = new FailureCommand(key, 1, sleepWindow);
HystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
assertTrue(cb.allowRequest());
assertFalse(cb.isOpen());
cmd1.execute();
HystrixCommand<Boolean> cmd2 = new FailureCommand(key, 1, sleepWindow);
cmd2.execute();
HystrixCommand<Boolean> cmd3 = new FailureCommand(key, 1, sleepWindow);
cmd3.execute();
HystrixCommand<Boolean> cmd4 = new TimeoutCommand(key, sleepWindow);
cmd4.execute();
// everything has failed in the test window so we should return false now
Thread.sleep(100);
System.out.println("ReqLog : " + HystrixRequestLog.getCurrentRequest().getExecutedCommandsAsString());
System.out.println("CircuitBreaker state 1 : " + cmd1.getMetrics().getHealthCounts());
assertTrue(cb.isOpen());
// wait for sleepWindow to pass
Thread.sleep(sleepWindow + 50);
// but the circuit should still be open
assertTrue(cb.isOpen());
// we should now allow 1 request, and upon success, should cause the circuit to be closed
HystrixCommand<Boolean> cmd5 = new SuccessCommand(key, 60, sleepWindow);
Observable<Boolean> asyncResult = cmd5.observe();
// and further requests are still blocked while the singleTest command is in flight
assertFalse(cb.allowRequest());
asyncResult.toBlocking().single();
// all requests should be open again
Thread.sleep(100);
System.out.println("CircuitBreaker state 2 : " + cmd1.getMetrics().getHealthCounts());
assertTrue(cb.allowRequest());
assertTrue(cb.allowRequest());
assertTrue(cb.allowRequest());
// and the circuit should be closed again
assertFalse(cb.isOpen());
} catch (Exception e) {
e.printStackTrace();
fail("Error occurred: " + e.getMessage());
}
}
/**
* Over a period of several 'windows' a single attempt will be made and fail and then finally succeed and close the circuit.
* <p>
* Ensure the circuit is kept open through the entire testing period and that only the single attempt in each window is made.
*/
@Test
public void testMultipleTimeWindowRetriesBeforeClosingCircuit() {
String key = "cmd-H";
try {
int sleepWindow = 200;
HystrixCommand<Boolean> cmd1 = new FailureCommand(key, 60);
HystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
assertTrue(cb.allowRequest());
assertFalse(cb.isOpen());
cmd1.execute();
HystrixCommand<Boolean> cmd2 = new FailureCommand(key, 1);
cmd2.execute();
HystrixCommand<Boolean> cmd3 = new FailureCommand(key, 1);
cmd3.execute();
HystrixCommand<Boolean> cmd4 = new TimeoutCommand(key);
cmd4.execute();
// everything has failed in the test window so we should return false now
System.out.println("!!!! 1: 4 failures, circuit will open on recalc");
Thread.sleep(100);
assertTrue(cb.isOpen());
// wait for sleepWindow to pass
System.out.println("!!!! 2: Sleep window starting where all commands fail-fast");
Thread.sleep(sleepWindow + 50);
System.out.println("!!!! 3: Sleep window over, should allow singleTest()");
// but the circuit should still be open
assertTrue(cb.isOpen());
// we should now allow 1 request, and upon failure, should not affect the circuit breaker, which should remain open
HystrixCommand<Boolean> cmd5 = new FailureCommand(key, 60);
Observable<Boolean> asyncResult5 = cmd5.observe();
System.out.println("!!!! 4: Kicked off the single-test");
// and further requests are still blocked while the singleTest command is in flight
assertFalse(cb.allowRequest());
System.out.println("!!!! 5: Confirmed that no other requests go out during single-test");
asyncResult5.toBlocking().single();
System.out.println("!!!! 6: SingleTest just completed");
// all requests should still be blocked, because the singleTest failed
assertFalse(cb.allowRequest());
assertFalse(cb.allowRequest());
assertFalse(cb.allowRequest());
// wait for sleepWindow to pass
System.out.println("!!!! 2nd sleep window START");
Thread.sleep(sleepWindow + 50);
System.out.println("!!!! 2nd sleep window over");
// we should now allow 1 request, and upon failure, should not affect the circuit breaker, which should remain open
HystrixCommand<Boolean> cmd6 = new FailureCommand(key, 60);
Observable<Boolean> asyncResult6 = cmd6.observe();
System.out.println("2nd singleTest just kicked off");
//and further requests are still blocked while the singleTest command is in flight
assertFalse(cb.allowRequest());
System.out.println("confirmed that 2nd singletest only happened once");
asyncResult6.toBlocking().single();
System.out.println("2nd singleTest now over");
// all requests should still be blocked, because the singleTest failed
assertFalse(cb.allowRequest());
assertFalse(cb.allowRequest());
assertFalse(cb.allowRequest());
// wait for sleepWindow to pass
Thread.sleep(sleepWindow + 50);
// but the circuit should still be open
assertTrue(cb.isOpen());
// we should now allow 1 request, and upon success, should cause the circuit to be closed
HystrixCommand<Boolean> cmd7 = new SuccessCommand(key, 60);
Observable<Boolean> asyncResult7 = cmd7.observe();
// and further requests are still blocked while the singleTest command is in flight
assertFalse(cb.allowRequest());
asyncResult7.toBlocking().single();
// all requests should be open again
assertTrue(cb.allowRequest());
assertTrue(cb.allowRequest());
assertTrue(cb.allowRequest());
// and the circuit should be closed again
assertFalse(cb.isOpen());
// and the circuit should be closed again
assertFalse(cb.isOpen());
} catch (Exception e) {
e.printStackTrace();
fail("Error occurred: " + e.getMessage());
}
}
/**
* When volume of reporting during a statistical window is lower than a defined threshold the circuit
* will not trip regardless of whatever statistics are calculated.
*/
@Test
public void testLowVolumeDoesNotTripCircuit() {
String key = "cmd-I";
try {
int sleepWindow = 200;
int lowVolume = 5;
HystrixCommand<Boolean> cmd1 = new FailureCommand(key, 60, sleepWindow, lowVolume);
HystrixCircuitBreaker cb = cmd1.circuitBreaker;
// this should start as allowing requests
assertTrue(cb.allowRequest());
assertFalse(cb.isOpen());
cmd1.execute();
HystrixCommand<Boolean> cmd2 = new FailureCommand(key, 1, sleepWindow, lowVolume);
cmd2.execute();
HystrixCommand<Boolean> cmd3 = new FailureCommand(key, 1, sleepWindow, lowVolume);
cmd3.execute();
HystrixCommand<Boolean> cmd4 = new FailureCommand(key, 1, sleepWindow, lowVolume);
cmd4.execute();
// even though it has all failed we won't trip the circuit because the volume is low
Thread.sleep(100);
assertTrue(cb.allowRequest());
assertFalse(cb.isOpen());
} catch (Exception e) {
e.printStackTrace();
fail("Error occurred: " + e.getMessage());
}
}
@Test
public void testUnsubscriptionDoesNotLeaveCircuitStuckHalfOpen() {
String key = "cmd-J";
try {
int sleepWindow = 200;
// fail
HystrixCommand<Boolean> cmd1 = new FailureCommand(key, 1, sleepWindow);
HystrixCommand<Boolean> cmd2 = new FailureCommand(key, 1, sleepWindow);
HystrixCommand<Boolean> cmd3 = new FailureCommand(key, 1, sleepWindow);
HystrixCommand<Boolean> cmd4 = new FailureCommand(key, 1, sleepWindow);
cmd1.execute();
cmd2.execute();
cmd3.execute();
cmd4.execute();
HystrixCircuitBreaker cb = cmd1.circuitBreaker;
// everything has failed in the test window so we should return false now
Thread.sleep(100);
assertFalse(cb.allowRequest());
assertTrue(cb.isOpen());
//this should occur after the sleep window, so get executed
//however, it is unsubscribed, so never updates state on the circuit-breaker
HystrixCommand<Boolean> cmd5 = new SuccessCommand(key, 5000, sleepWindow);
//wait for sleep window to pass
Thread.sleep(sleepWindow + 50);
Observable<Boolean> o = cmd5.observe();
Subscription s = o.subscribe();
s.unsubscribe();
//wait for 10 sleep windows, then try a successful command. this should return the circuit to CLOSED
Thread.sleep(10 * sleepWindow);
HystrixCommand<Boolean> cmd6 = new SuccessCommand(key, 1, sleepWindow);
cmd6.execute();
Thread.sleep(100);
assertTrue(cb.allowRequest());
assertFalse(cb.isOpen());
} catch (Exception e) {
e.printStackTrace();
fail("Error occurred: " + e.getMessage());
}
}
/**
* Utility method for creating {@link HystrixCommandMetrics} for unit tests.
*/
private static HystrixCommandMetrics getMetrics(HystrixCommandProperties.Setter properties) {
return HystrixCommandMetrics.getInstance(CommandKeyForUnitTest.KEY_ONE, CommandOwnerForUnitTest.OWNER_ONE, ThreadPoolKeyForUnitTest.THREAD_POOL_ONE, HystrixCommandPropertiesTest.asMock(properties));
}
/**
* Utility method for creating {@link HystrixCommandMetrics} for unit tests.
*/
private static HystrixCommandMetrics getMetrics(HystrixCommandKey commandKey, HystrixCommandProperties.Setter properties) {
return HystrixCommandMetrics.getInstance(commandKey, CommandOwnerForUnitTest.OWNER_ONE, ThreadPoolKeyForUnitTest.THREAD_POOL_ONE, HystrixCommandPropertiesTest.asMock(properties));
}
/**
* Utility method for creating {@link HystrixCircuitBreaker} for unit tests.
*/
private static HystrixCircuitBreaker getCircuitBreaker(HystrixCommandKey key, HystrixCommandGroupKey commandGroup, HystrixCommandMetrics metrics, HystrixCommandProperties.Setter properties) {
return new HystrixCircuitBreakerImpl(key, commandGroup, HystrixCommandPropertiesTest.asMock(properties), metrics);
}
private static enum CommandOwnerForUnitTest implements HystrixCommandGroupKey {
OWNER_ONE, OWNER_TWO
}
private static enum ThreadPoolKeyForUnitTest implements HystrixThreadPoolKey {
THREAD_POOL_ONE, THREAD_POOL_TWO
}
private static enum CommandKeyForUnitTest implements HystrixCommandKey {
KEY_ONE, KEY_TWO
}
// ignoring since this never ends ... useful for testing https://github.com/Netflix/Hystrix/issues/236
@Ignore
@Test
public void testSuccessClosesCircuitWhenBusy() throws InterruptedException {
HystrixPlugins.getInstance().registerCommandExecutionHook(new MyHystrixCommandExecutionHook());
try {
performLoad(200, 0, 40);
performLoad(250, 100, 40);
performLoad(600, 0, 40);
} finally {
Hystrix.reset();
}
}
void performLoad(int totalNumCalls, int errPerc, int waitMillis) {
Random rnd = new Random();
for (int i = 0; i < totalNumCalls; i++) {
//System.out.println(i);
try {
boolean err = rnd.nextFloat() * 100 < errPerc;
TestCommand cmd = new TestCommand(err);
cmd.execute();
} catch (Exception e) {
//System.err.println(e.getMessage());
}
try {
Thread.sleep(waitMillis);
} catch (InterruptedException e) {
}
}
}
public class TestCommand extends HystrixCommand<String> {
boolean error;
public TestCommand(final boolean error) {
super(HystrixCommandGroupKey.Factory.asKey("group"));
this.error = error;
}
@Override
protected String run() throws Exception {
if (error) {
throw new Exception("forced failure");
} else {
return "success";
}
}
@Override
protected String getFallback() {
if (isFailedExecution()) {
return getFailedExecutionException().getMessage();
} else {
return "other fail reason";
}
}
}
private class Command extends HystrixCommand<Boolean> {
private final boolean shouldFail;
private final boolean shouldFailWithBadRequest;
private final long latencyToAdd;
public Command(String commandKey, boolean shouldFail, boolean shouldFailWithBadRequest, long latencyToAdd, int sleepWindow, int requestVolumeThreshold) {
super(Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey("Command")).andCommandKey(HystrixCommandKey.Factory.asKey(commandKey)).
andCommandPropertiesDefaults(HystrixCommandPropertiesTest.getUnitTestPropertiesSetter().
withExecutionTimeoutInMilliseconds(500).
withCircuitBreakerRequestVolumeThreshold(requestVolumeThreshold).
withCircuitBreakerSleepWindowInMilliseconds(sleepWindow)));
this.shouldFail = shouldFail;
this.shouldFailWithBadRequest = shouldFailWithBadRequest;
this.latencyToAdd = latencyToAdd;
}
public Command(String commandKey, boolean shouldFail, long latencyToAdd) {
this(commandKey, shouldFail, false, latencyToAdd, 200, 1);
}
@Override
protected Boolean run() throws Exception {
Thread.sleep(latencyToAdd);
if (shouldFail) {
throw new RuntimeException("induced failure");
}
if (shouldFailWithBadRequest) {
throw new HystrixBadRequestException("bad request");
}
return true;
}
@Override
protected Boolean getFallback() {
return false;
}
}
private class SuccessCommand extends Command {
SuccessCommand(String commandKey, long latencyToAdd) {
super(commandKey, false, latencyToAdd);
}
SuccessCommand(String commandKey, long latencyToAdd, int sleepWindow) {
super(commandKey, false, false, latencyToAdd, sleepWindow, 1);
}
}
private class FailureCommand extends Command {
FailureCommand(String commandKey, long latencyToAdd) {
super(commandKey, true, latencyToAdd);
}
FailureCommand(String commandKey, long latencyToAdd, int sleepWindow) {
super(commandKey, true, false, latencyToAdd, sleepWindow, 1);
}
FailureCommand(String commandKey, long latencyToAdd, int sleepWindow, int requestVolumeThreshold) {
super(commandKey, true, false, latencyToAdd, sleepWindow, requestVolumeThreshold);
}
}
private class TimeoutCommand extends Command {
TimeoutCommand(String commandKey) {
super(commandKey, false, 2000);
}
TimeoutCommand(String commandKey, int sleepWindow) {
super(commandKey, false, false, 2000, sleepWindow, 1);
}
}
private class BadRequestCommand extends Command {
BadRequestCommand(String commandKey, long latencyToAdd) {
super(commandKey, false, true, latencyToAdd, 200, 1);
}
BadRequestCommand(String commandKey, long latencyToAdd, int sleepWindow) {
super(commandKey, false, true, latencyToAdd, sleepWindow, 1);
}
}
public class MyHystrixCommandExecutionHook extends HystrixCommandExecutionHook {
@Override
public <T> T onComplete(final HystrixInvokable<T> command, final T response) {
logHC(command, response);
return super.onComplete(command, response);
}
private int counter = 0;
private <T> void logHC(HystrixInvokable<T> command, T response) {
if(command instanceof HystrixInvokableInfo) {
HystrixInvokableInfo<T> commandInfo = (HystrixInvokableInfo<T>)command;
HystrixCommandMetrics metrics = commandInfo.getMetrics();
System.out.println("cb/error-count/%/total: "
+ commandInfo.isCircuitBreakerOpen() + " "
+ metrics.getHealthCounts().getErrorCount() + " "
+ metrics.getHealthCounts().getErrorPercentage() + " "
+ metrics.getHealthCounts().getTotalRequests() + " => " + response + " " + commandInfo.getExecutionEvents());
}
}
}
}