Skip to content
Browse files

Don't throw ShardBlackHoleExceptions when a client has hit its error …

…limit.
  • Loading branch information...
1 parent 5d9cb2e commit b953532d72010c1acaea61d5caf3352275fbda1d @rcohen rcohen committed Aug 31, 2011
View
4 config/development.scala
@@ -29,7 +29,6 @@ new HaplocheirusConfig {
val redisConfig = new RedisConfig {
val poolHealthTrackerConfig = new RedisPoolHealthTrackerConfig {
- val clientErrorLimit = 100
val autoDisableErrorLimit = 200
val autoDisableDuration = 60.seconds
}
@@ -42,6 +41,7 @@ new HaplocheirusConfig {
val timeoutMsec = 200
val keysTimeoutMsec = 5000
val expirationHours = 24 * 21
+ val clientErrorLimit = 100
}
val writePoolConfig = new RedisPoolConfig {
@@ -52,6 +52,7 @@ new HaplocheirusConfig {
val timeoutMsec = 200
val keysTimeoutMsec = 5000
val expirationHours = 24 * 21
+ val clientErrorLimit = 100
}
val slowPoolConfig = new RedisPoolConfig {
@@ -62,6 +63,7 @@ new HaplocheirusConfig {
val timeoutMsec = 200
val keysTimeoutMsec = 5000
val expirationHours = 24 * 21
+ val clientErrorLimit = 100
}
val rangeQueryPageSize = 20
View
4 config/production.scala
@@ -67,7 +67,6 @@ new HaplocheirusConfig {
val redisConfig = new RedisConfig {
val poolHealthTrackerConfig = new RedisPoolHealthTrackerConfig {
- val clientErrorLimit = 250
val autoDisableErrorLimit = 500
val autoDisableDuration = 60.seconds
}
@@ -80,6 +79,7 @@ new HaplocheirusConfig {
val timeoutMsec = 5
val keysTimeoutMsec = 5000
val expirationHours = 24 * 21
+ val clientErrorLimit = 250
}
val writePoolConfig = new RedisPoolConfig {
@@ -90,6 +90,7 @@ new HaplocheirusConfig {
val timeoutMsec = 5
val keysTimeoutMsec = 5000
val expirationHours = 24 * 21
+ val clientErrorLimit = 250
}
val slowPoolConfig = new RedisPoolConfig {
@@ -100,6 +101,7 @@ new HaplocheirusConfig {
val timeoutMsec = 50
val keysTimeoutMsec = 5000
val expirationHours = 24 * 21
+ val clientErrorLimit = 250
}
val rangeQueryPageSize = 20
View
4 config/test.scala
@@ -29,7 +29,6 @@ new HaplocheirusConfig {
val redisConfig = new RedisConfig {
val poolHealthTrackerConfig = new RedisPoolHealthTrackerConfig {
- val clientErrorLimit = 100
val autoDisableErrorLimit = 200
val autoDisableDuration = 60.seconds
}
@@ -42,6 +41,7 @@ new HaplocheirusConfig {
val timeoutMsec = 200
val keysTimeoutMsec = 5000
val expirationHours = 24
+ val clientErrorLimit = 100
}
val writePoolConfig = new RedisPoolConfig {
@@ -52,6 +52,7 @@ new HaplocheirusConfig {
val timeoutMsec = 200
val keysTimeoutMsec = 5000
val expirationHours = 24
+ val clientErrorLimit = 100
}
val slowPoolConfig = new RedisPoolConfig {
@@ -62,6 +63,7 @@ new HaplocheirusConfig {
val timeoutMsec = 200
val keysTimeoutMsec = 5000
val expirationHours = 24
+ val clientErrorLimit = 100
}
val rangeQueryPageSize = 20
View
3 src/main/scala/com/twitter/haplocheirus/HaplocheirusConfig.scala
@@ -20,7 +20,6 @@ trait AdminConfig {
}
trait RedisPoolHealthTrackerConfig {
- def clientErrorLimit: Int
def autoDisableErrorLimit: Int
def autoDisableDuration: Duration
}
@@ -37,6 +36,8 @@ trait RedisPoolConfig {
// expiration on timelines
def expirationHours: Int
+
+ def clientErrorLimit: Int
}
trait RedisConfig {
View
26 src/main/scala/com/twitter/haplocheirus/RedisPool.scala
@@ -34,10 +34,7 @@ class RedisPoolHealthTracker(config: RedisPoolHealthTrackerConfig) {
}
if (client ne null) {
- val clientC = client.errorCount.incrementAndGet
- if (clientC > config.clientErrorLimit) {
- log.error("Too many errors on client for %s", hostname)
- }
+ client.errorCount.incrementAndGet
}
}
@@ -52,16 +49,16 @@ class RedisPoolHealthTracker(config: RedisPoolHealthTrackerConfig) {
client.errorCount.set(0)
}
- def isErrored(hostname: String, client: PipelinedRedisClient): Boolean = {
+ def isErrored(hostname: String): Boolean = {
val timeout = concurrentDisabledMap.get(hostname)
- val hostnameErrored = if (!(timeout eq null)) {
+ if (!(timeout eq null)) {
if (Time.now < timeout) {
true
} else {
try {
- concurrentDisabledMap.remove(hostname)
log.error("Reenabling %s", hostname)
- countNonError(hostname, client) // To remove from the error map
+ concurrentErrorMap.remove(hostname)
+ concurrentDisabledMap.remove(hostname)
} catch {
case e: NullPointerException => {}
}
@@ -70,12 +67,6 @@ class RedisPoolHealthTracker(config: RedisPoolHealthTrackerConfig) {
} else {
false
}
- val clientErrored = if ((client ne null) && (client.errorCount.get > config.clientErrorLimit)) {
- true
- } else {
- false
- }
- hostnameErrored || clientErrored
}
}
@@ -104,10 +95,13 @@ class RedisPool(name: String, healthTracker: RedisPoolHealthTracker, config: Red
val server = poolIndexGenerator.nextInt(config.poolSize)
var client = serverPool(server).get(hostname);
- if (healthTracker.isErrored(shardInfo.hostname, client)) {
- if ((client ne null) && client.alive) {
+ if ((client ne null) && (client.errorCount.get > config.clientErrorLimit)) {
+ if (client.alive) {
throwAway(hostname, client)
}
+ }
+
+ if (healthTracker.isErrored(shardInfo.hostname)) {
throw new ShardBlackHoleException(shardInfo.id)
}

0 comments on commit b953532

Please sign in to comment.
Something went wrong with that request. Please try again.