Merge pull request #8 from adhorn/failurerate

adding possibility for a controlled failure rate

Author: adhorn

README.md

@@ -10,14 +10,16 @@
 * Using for SSM Parameter Store to control the experiment using ```isEnabled```
 * Per Lambda function injection control using Environment variable (```FAILURE_INJECTION_PARAM```) (thanks to Gunnar Grosh)
 * Support for Serverless Framework using ```sls deploy``` (thanks to Gunnar Grosh)
+* Support for adding rate of failure using ```rate```. (Default rate = 1)
 
 ### Parameter Store Object
 ```json
 { 
     "delay": 200,
     "isEnabled": true,
     "error_code": 404,
-    "exception_msg": "I FAILED"
+    "exception_msg": "I FAILED",
+    "rate": 0.5
 }
 ```
 Deploy the chaos config in paramater store.

python/chaos_lib.py

@@ -27,11 +27,9 @@ def get_config(config_key):
     param = SSMParameter(os.environ['FAILURE_INJECTION_PARAM'])
     try:
         value = json.loads(param.value)
-        isEnabled = value["isEnabled"]
-        if not isEnabled:
+        if not value["isEnabled"]:
             return 0
-        key_ = value[config_key]
-        return key_
+        return value[config_key], value.get('rate', 1)
     except InvalidParameterError as e:
         # key does not exist in SSM
         raise InvalidParameterError("{} does not exist in SSM".format(e))
@@ -42,18 +40,14 @@ def get_config(config_key):
 
 def corrupt_delay(func):
     def latency(*args, **kw):
-        delay = get_config('delay')
-        print(delay)
+        delay, rate = get_config('delay')
+        print("delay: {0}, rate: {1}".format(delay, rate))
         start = time.time()
-        # if delay exist, delaying with that value
-        if delay > 0:
-            time.sleep(delay / 1000.0)
-        # if delay = -1 make random delays
-        elif delay < 0:
-            # add latency approx 50% of the time
-            if random.random() > 0.5:
-                # random sleep time between 1 and 10 seconds
-                time.sleep(random.randint(1, 10))
+        # if delay and rate exist, delaying with that value at that rate
+        if delay > 0 and rate >= 0:
+            # add latency approx rate% of the time
+            if random.random() <= rate:
+                time.sleep(delay / 1000.0)
 
         result = func(*args, **kw)
         end = time.time()
@@ -69,19 +63,27 @@ def latency(*args, **kw):
 def corrupt_expection(func):
     def wrapper(*args, **kwargs):
         result = func(*args, **kwargs)
-        exception_msg = get_config('exception_msg')
-        print("exception_msg from config {}".format(exception_msg))
-        print("corrupting now")
-        raise Exception(exception_msg)
+        exception_msg, rate = get_config('exception_msg')
+        print("exception_msg from config {0} with a rate of {1}".format(exception_msg, rate))
+        # add injection approx rate% of the time
+        if random.random() <= rate:
+            print("corrupting now")
+            raise Exception(exception_msg)
+        else:
+            return result
     return wrapper
 
 
 def corrupt_statuscode(func):
     def wrapper(*args, **kwargs):
         result = func(*args, **kwargs)
-        error_code = get_config('error_code')
-        print("Error from config {}".format(error_code))
-        print("corrupting now")
-        result['statusCode'] = error_code
-        return result
+        error_code, rate = get_config('error_code')
+        print("Error from config {0} at a rate of {1}".format(error_code, rate))
+        # add injection approx rate% of the time
+        if random.random() <= rate:
+            print("corrupting now")
+            result['statusCode'] = error_code
+            return result
+        else:
+            return result
     return wrapper

python/test.py

@@ -23,7 +23,7 @@ class TestStringMethods(unittest.TestCase):
     def setUp(self):
         os.environ['FAILURE_INJECTION_PARAM'] = 'test.config'
         client.put_parameter(
-            Value="{ \"delay\": 200, \"isEnabled\": true, \"error_code\": 404, \"exception_msg\": \"I FAILED\"}",
+            Value="{ \"delay\": 200, \"isEnabled\": true, \"error_code\": 404, \"exception_msg\": \"I FAILED\", \"rate\": 0.5 }",
             Name='test.config',
             Type='String',
             Overwrite=True
@@ -35,18 +35,23 @@ def tearDown(self):
 
     @ignore_warnings
     def test_get_config(self):
-        isEnabled = get_config('isEnabled')
+        isEnabled, rate = get_config('isEnabled')
         self.assertEqual(isEnabled, True or False)
+        self.assertEqual(rate, 0.5)
 
     @ignore_warnings
     def test_get_config_delay(self):
-        delay = get_config('delay')
+        delay, rate = get_config('delay')
         self.assertEqual(delay, 200)
+        self.assertEqual(rate, 0.5)
+
 
     @ignore_warnings
     def test_get_config_error_code(self):
-        delay = get_config('error_code')
+        delay, rate = get_config('error_code')
         self.assertEqual(delay, 404)
+        self.assertEqual(rate, 0.5)
+
 
     @ignore_warnings
     def test_get_config_bad_key(self):