Skip to content
Browse files

unwrap exceptions thrown by jobs, and don't log the stack trace on "n…

…ormal" (timeout or transient) exceptions.
  • Loading branch information...
1 parent 34b2ca9 commit ba8fc50821a4b3b13a47595c27c2d21ad166e424 Robey Pointer committed
Showing with 33 additions and 13 deletions.
  1. +33 −13 src/main/scala/com/twitter/gizzard/scheduler/JobScheduler.scala
View
46 src/main/scala/com/twitter/gizzard/scheduler/JobScheduler.scala
@@ -1,13 +1,15 @@
package com.twitter.gizzard.scheduler
+import java.lang.reflect.UndeclaredThrowableException
import java.util.Random
+import java.util.concurrent.ExecutionException
import com.twitter.ostrich.{BackgroundProcess, Stats}
import com.twitter.xrayspecs.Duration
import com.twitter.xrayspecs.TimeConversions._
import net.lag.configgy.ConfigMap
import net.lag.kestrel.PersistentQueue
import net.lag.logging.Logger
-import shards.ShardRejectedOperationException
+import shards.{NormalShardException, ShardRejectedOperationException}
object JobScheduler {
/**
@@ -154,6 +156,18 @@ class JobScheduler[J <: Job](val name: String,
}
}
+ private def unwrapException(exception: Throwable): Throwable = {
+ exception match {
+ case e: ExecutionException =>
+ unwrapException(e.getCause)
+ case e: UndeclaredThrowableException =>
+ // fondly known as JavaOutrageException
+ unwrapException(e.getCause)
+ case e =>
+ e
+ }
+ }
+
// hook to let unit tests stub out threads.
protected def processWork() {
process()
@@ -166,18 +180,24 @@ class JobScheduler[J <: Job](val name: String,
job()
Stats.incr("job-success-count")
} catch {
- case e: ShardRejectedOperationException =>
- Stats.incr("job-darkmoded-count")
- errorQueue.put(job)
- case e =>
- Stats.incr("job-error-count")
- log.error(e, "Error in Job: %s - %s", job, e)
- job.errorCount += 1
- job.errorMessage = e.toString
- if (job.errorCount > errorLimit) {
- badJobQueue.foreach { _.put(job) }
- } else {
- errorQueue.put(job)
+ case e: Throwable =>
+ unwrapException(e) match {
+ case e: ShardRejectedOperationException =>
+ Stats.incr("job-darkmoded-count")
+ errorQueue.put(job)
+ case e: NormalShardException =>
+ Stats.incr("job-error-count")
+ log.error("Error in Job: %s - %s", job, e)
+ case e =>
+ Stats.incr("job-error-count")
+ log.error(e, "Error in Job: %s - %s", job, e)
+ job.errorCount += 1
+ job.errorMessage = e.toString
+ if (job.errorCount > errorLimit) {
+ badJobQueue.foreach { _.put(job) }
+ } else {
+ errorQueue.put(job)
+ }
}
}
ticket.ack()

0 comments on commit ba8fc50

Please sign in to comment.
Something went wrong with that request. Please try again.