Merge 461340c into 0992a25

TreeGateway · Jan 25, 2018 · e376247 · e376247
2 parents 0992a25 + 461340c
commit e376247
Show file tree

Hide file tree

Showing 5 changed files with 50 additions and 9 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "tree-gateway",
-  "version": "1.5.5",
+  "version": "1.5.6",
   "homepage": "http://treegateway.org",
   "description": "The Tree Gateway API Gateway",
   "author": "Thiago da Rosa de Bustamante <trbustamante@gmail.com>",

diff --git a/src/circuitbreaker/circuit-breaker.ts b/src/circuitbreaker/circuit-breaker.ts
@@ -45,7 +45,9 @@ export class ApiCircuitBreaker {
                 maxFailures: (cbConfig.maxFailures || 10),
                 rejectMessage: (cbConfig.rejectMessage || 'Service unavailable'),
                 rejectStatusCode: (cbConfig.rejectStatusCode || 503),
-                stateHandler: new RedisStateHandler(cbStateID, getMilisecondsInterval(cbConfig.resetTimeout, 120000)),
+                stateHandler: new RedisStateHandler(cbStateID,
+                                    getMilisecondsInterval(cbConfig.resetTimeout, 120000),
+                                    getMilisecondsInterval(cbConfig.timeWindow)),
                 timeout: getMilisecondsInterval(cbConfig.timeout, 30000),
                 timeoutMessage: (cbConfig.timeoutMessage || 'Operation timeout'),
                 timeoutStatusCode: (cbConfig.timeoutStatusCode || 504)

diff --git a/src/circuitbreaker/redis-state-handler.ts b/src/circuitbreaker/redis-state-handler.ts
@@ -20,14 +20,16 @@ export class RedisStateHandler implements StateHandler {
     private state: State;
     halfOpenCallPending: boolean;
     private resetTimeout: number;
+    private timeWindow: number;
 
-    constructor(id: string, resetTimeout: number) {
+    constructor(id: string, resetTimeout: number, timeWindow?: number) {
         this.id = id;
         this.resetTimeout = resetTimeout;
+        this.timeWindow = timeWindow;
     }
 
     initialState() {
-        this.database.redisClient.hget(CircuitBreakerKeys.CIRCUIT_BREAKER_STATE, this.id)
+        this.database.redisClient.get(this.getRedisStateKey())
             .then((state: any) => {
                 if (state === 'open') {
                     this.openState();
@@ -73,11 +75,34 @@ export class RedisStateHandler implements StateHandler {
         }
 
         this.state = State.HALF_OPEN;
+        this.halfOpenCallPending = false;
         return true;
     }
 
     incrementFailures(): Promise<number> {
-        return this.database.redisClient.hincrby(CircuitBreakerKeys.CIRCUIT_BREAKER_FAILURES, this.id, 1);
+        const rdskey = this.getRedisFailuresKey();
+        if (this.timeWindow) {
+            return new Promise<number>((resolve, reject) => {
+                this.database.redisClient.multi()
+                .incr(rdskey)
+                .ttl(rdskey)
+                .exec((err, replies) => {
+                    if (err) {
+                        return reject(err);
+                    }
+
+                    // if this is new or has no expire
+                    if (replies[0][1] === 1 || replies[1][1] === -1) {
+                        // then expire it after the timeout
+                        this.database.redisClient.expire(rdskey, this.timeWindow);
+                    }
+
+                    resolve(replies[0][1]);
+                });
+            });
+        } else {
+            return this.database.redisClient.incr(rdskey);
+        }
     }
 
     onStateChanged(state: string) {
@@ -99,6 +124,14 @@ export class RedisStateHandler implements StateHandler {
         }
     }
 
+    private getRedisFailuresKey() {
+        return `${CircuitBreakerKeys.CIRCUIT_BREAKER_FAILURES}:${this.id}`;
+    }
+
+    private getRedisStateKey() {
+        return `${CircuitBreakerKeys.CIRCUIT_BREAKER_STATE}:${this.id}`;
+    }
+
     private openState(): boolean {
         if (this.state === State.OPEN) {
             return false;
@@ -128,7 +161,7 @@ export class RedisStateHandler implements StateHandler {
             this.logger.debug(`Notifying cluster that circuit for API ${this.id} is open`);
         }
         return this.database.redisClient.multi()
-            .hset(CircuitBreakerKeys.CIRCUIT_BREAKER_STATE, this.id, 'open')
+            .set(this.getRedisStateKey(), 'open')
             .publish(ConfigTopics.CIRCUIT_CHANGED, JSON.stringify({ state: 'open', id: this.id }))
             .exec();
     }
@@ -139,8 +172,8 @@ export class RedisStateHandler implements StateHandler {
         }
 
         return this.database.redisClient.multi()
-            .hset(CircuitBreakerKeys.CIRCUIT_BREAKER_STATE, this.id, 'close')
-            .hdel(CircuitBreakerKeys.CIRCUIT_BREAKER_FAILURES, this.id)
+            .set(this.getRedisStateKey(), 'close')
+            .del(this.getRedisFailuresKey(), this.id)
             .publish(ConfigTopics.CIRCUIT_CHANGED, JSON.stringify({ state: 'close', id: this.id }))
             .exec();
     }

diff --git a/src/config/circuit-breaker.ts b/src/config/circuit-breaker.ts
@@ -22,6 +22,11 @@ export interface CircuitBreakerConfig {
      * Defaults to '2 minutes'
      */
     resetTimeout?: string | number;
+    /**
+     * Define a time window to count the api failures. If defined, the gateway will reset the total failures count
+     * by each period of time defined here.
+     */
+    timeWindow?: string | number;
     /**
      * When the failure counter reaches a maxFailures count, the breaker is tripped into Open state.
      */
@@ -121,6 +126,7 @@ export let circuitBreakerConfigValidatorSchema = Joi.object().keys({
     rejectStatusCode: Joi.number(),
     resetTimeout: Joi.alternatives([Joi.string(), Joi.number().positive()]),
     statsConfig: statsConfigValidatorSchema,
+    timeWindow: Joi.alternatives([Joi.string(), Joi.number().positive()]),
     timeout: Joi.alternatives([Joi.string(), Joi.number().positive()]),
     timeoutMessage: Joi.string(),
     timeoutStatusCode: Joi.number()