Merge pull request #14 from luizalabs/circuit_breaker

Circuit breaker

django_toolkit/fallbacks/circuit_breaker.py

@@ -0,0 +1,77 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class CircuitBreaker:
+
+    def __init__(
+        self,
+        cache,
+        failure_cache_key,
+        max_failures,
+        max_failure_exception,
+        max_failure_timeout=None,
+        circuit_timeout=None,
+        catch_exceptions=None,
+    ):
+        self.cache = cache
+        self.failure_cache_key = failure_cache_key
+        self.max_failure_timeout = max_failure_timeout
+        self.circuit_timeout = circuit_timeout
+        self.circuit_cache_key = 'circuit_{}'.format(failure_cache_key)
+        self.max_failure_exception = max_failure_exception
+        self.catch_exceptions = catch_exceptions or (Exception,)
+        self.max_failures = max_failures
+
+    @property
+    def is_circuit_open(self):
+        return self.cache.get(self.circuit_cache_key) or False
+
+    @property
+    def total_failures(self):
+        return self.cache.get(self.failure_cache_key) or 0
+
+    def open_circuit(self):
+        logger.critical(
+            'Open circuit for {failure_cache_key} {cicuit_cache_key}'.format(
+                failure_cache_key=self.failure_cache_key,
+                cicuit_cache_key=self.circuit_cache_key
+            )
+        )
+
+        self.cache.set(self.circuit_cache_key, True, self.circuit_timeout)
+
+    def __enter__(self):
+        if self.is_circuit_open:
+            raise self.max_failure_exception
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if exc_type in self.catch_exceptions:
+
+            self._increase_failure_count()
+
+            if self.total_failures >= self.max_failures:
+                self.open_circuit()
+
+                logger.info(
+                    'Max failures exceeded by: {}'.format(
+                        self.failure_cache_key
+                    )
+                )
+
+                raise self.max_failure_exception
+
+    def _increase_failure_count(self):
+        self.cache.add(self.failure_cache_key, 0, self.max_failure_timeout)
+        total = self.cache.incr(self.failure_cache_key)
+
+        logger.info(
+            'Increase failure for: {key} - '
+            'max failures {max_failures} - '
+            'total {total}'.format(
+                key=self.failure_cache_key,
+                max_failures=self.max_failures,
+                total=total
+            )
+        )

docs/fallbacks.md

@@ -0,0 +1,60 @@
+# Fallbacks
+
+### Circuit Breaker
+
+A simple implementation of [Circuit Breaker](http://martinfowler.com/bliki/CircuitBreaker.html) pattern.
+
+#### Arguments
+
+`cache`
+
+Django cache object.
+
+`failure_cache_key`
+
+Cache key where the number of errors is incremented.
+
+`max_failures`
+
+Maximum number of errors.
+
+`max_failure_exception`
+
+Exception raised when exceded maximum number of errors and when circuit is open.
+
+`max_failure_timeout`
+
+This value is set on first error. It is used to validate number of errors by time.
+
+`circuit_timeout`
+
+Time that the circuit will be open.
+
+`catch_exceptions`
+
+List of exception that is catched to incrase number of errors.
+
+
+#### Example
+
+```python
+from django_toolkit.failures.circuit_breaker import CircuitBreaker
+from django.core.cache import caches
+
+cache = caches['default']
+
+class MyException(Exception):
+    pass
+
+
+with CircuitBreaker(
+    cache=cache,
+    failure_cache_key='failure_cache_key',
+    max_failures=1000,
+    max_failure_exception=MyException,
+    max_failure_timeout=3600,
+    circuit_timeout=5000,
+    catch_exceptions=(ValueError, StandardError, LookupError),
+) as circuit_breaker:
+    assert not circuit_breaker.is_circuit_open
+```

docs/index.md

@@ -10,5 +10,6 @@ This package includes the utility modules:
 * [logs](logs)
 * [middlewares](middlewares)
 * [shortcuts](shortcuts)
+* [fallbacks](fallbacks)
 
 [django-website]: https://www.djangoproject.com/

requirements-dev.txt

@@ -1,12 +1,12 @@
 -e .
 bumpversion==0.5.3
 django-oauth-toolkit==0.10.0
-djangorestframework==3.4.7
+djangorestframework==3.5.2
 flake8==3.0.4
 isort==4.2.5
-mixer==5.5.7
+mixer==5.5.8
 mkdocs==0.15.3
 mock==2.0.0
-pytest-cov==2.3.1
+pytest-cov==2.4.0
 pytest-django==3.0.0
-pytest==3.0.2
+pytest==3.0.3

tests/fallbacks/test_circuit_breaker.py

@@ -0,0 +1,94 @@
+import pytest
+from django.core.cache import caches
+
+from django_toolkit.fallbacks.circuit_breaker import CircuitBreaker
+
+cache = caches['default']
+
+
+class MyException(Exception):
+    pass
+
+
+def success_function():
+    return True
+
+
+def fail_function():
+    raise ValueError()
+
+
+class TestCircuitBreaker:
+
+    def test_success_result(self):
+        with CircuitBreaker(
+            cache=cache,
+            failure_cache_key='success',
+            max_failures=1,
+            max_failure_exception=None,
+            catch_exceptions=None,
+        ):
+            success_function()
+
+    def test_should_raise_error_when_max_failures_is_exceeded(self):
+        with pytest.raises(MyException):
+            with CircuitBreaker(
+                cache=cache,
+                failure_cache_key='fail',
+                max_failures=0,
+                max_failure_exception=MyException,
+                catch_exceptions=(ValueError,),
+            ):
+                fail_function()
+
+    def test_should_increase_fail_cache_count(self):
+        failure_cache_key = 'fail_count'
+
+        cache.set(failure_cache_key, 171)
+
+        with pytest.raises(ValueError):
+            with CircuitBreaker(
+                cache=cache,
+                failure_cache_key=failure_cache_key,
+                max_failures=5000,
+                max_failure_exception=MyException,
+                catch_exceptions=(ValueError,),
+            ):
+                fail_function()
+
+        assert cache.get(failure_cache_key, 172)
+
+    def test_should_open_circuit_when_max_failures_exceeds(self):
+        failure_cache_key = 'circuit'
+
+        cache.set(failure_cache_key, 1)
+
+        with pytest.raises(MyException):
+            with CircuitBreaker(
+                cache=cache,
+                failure_cache_key=failure_cache_key,
+                max_failures=2,
+                max_failure_exception=MyException,
+                catch_exceptions=(ValueError,),
+            ) as circuit_breaker:
+                fail_function()
+
+            assert circuit_breaker.is_circuit_open
+
+        assert cache.get(failure_cache_key, 2)
+
+    def test_should_raise_exception_when_circuit_is_open(self):
+
+        cache.set('circuit_circuit_open', True)
+
+        with pytest.raises(MyException):
+            with CircuitBreaker(
+                cache=cache,
+                failure_cache_key='circuit_open',
+                max_failures=10,
+                max_failure_exception=MyException,
+                catch_exceptions=(ValueError,),
+            ) as circuit_breaker:
+                success_function()
+
+            assert circuit_breaker.is_circuit_open