Merge branch 'master' of github.com:twitter/gizzard

Conflicts:
	src/main/scala/com/twitter/gizzard/shards/RoutingNode.scala

.gitignore

@@ -12,3 +12,6 @@ gizzard.tmproj
 exception.log
 .ivyjars
 .scala_dependencies
+*.iml
+.idea/
+*.log

project/build.properties

@@ -3,6 +3,6 @@
 project.organization=com.twitter
 project.name=gizzard
 sbt.version=0.7.4
-project.version=3.0.0-beta7
+project.version=3.0.0-beta10-SNAPSHOT
 build.scala.versions=2.8.1
 project.initialize=false

project/build/GizzardProject.scala

@@ -9,7 +9,7 @@ with SubversionPublisher {
   override def filterScalaJars = false
   val scalaTools = "org.scala-lang" % "scala-compiler" % "2.8.1"
 
-  val querulous  = "com.twitter" % "querulous" % "2.3.8"
+  val querulous  = "com.twitter" % "querulous" % "2.3.10"
 
   //val kestrel     = "net.lag" % "kestrel" % "1.2.7"
   // remove when moved to libkestrel

src/main/scala/com/twitter/gizzard/proxy/SqlExceptionWrappingProxy.scala

@@ -3,7 +3,7 @@ package com.twitter.gizzard.proxy
 import java.sql.SQLException
 import scala.reflect.Manifest
 import com.mysql.jdbc.exceptions.MySQLTransientException
-import com.twitter.querulous.database.SqlDatabaseTimeoutException
+import com.twitter.querulous.database.{PoolTimeoutException, PoolEmptyException, SqlDatabaseTimeoutException}
 import com.twitter.querulous.query.SqlQueryTimeoutException
 import com.twitter.querulous.evaluator.{QueryEvaluator, QueryEvaluatorProxy}
 import com.twitter.gizzard.shards._
@@ -15,6 +15,10 @@ class SqlExceptionWrappingProxy(shardId: ShardId) extends ExceptionHandlingProxy
       throw new ShardTimeoutException(e.timeout, shardId, e)
     case e: SqlDatabaseTimeoutException =>
       throw new ShardDatabaseTimeoutException(e.timeout, shardId, e)
+    case e: PoolTimeoutException =>
+      throw new NormalShardException(e.toString, shardId, e)
+    case e: PoolEmptyException =>
+      throw new NormalShardException(e.toString, shardId, e)
     case e: MySQLTransientException =>
       throw new NormalShardException(e.toString, shardId, null)
     case e: SQLException =>
@@ -36,6 +40,10 @@ class SqlExceptionWrappingProxyFactory[T <: AnyRef : Manifest](id: ShardId) exte
       throw new ShardTimeoutException(e.timeout, id, e)
     case e: SqlDatabaseTimeoutException =>
       throw new ShardDatabaseTimeoutException(e.timeout, id, e)
+    case e: PoolTimeoutException =>
+      throw new NormalShardException(e.toString, id, e)
+    case e: PoolEmptyException =>
+      throw new NormalShardException(e.toString, id, e)
     case e: MySQLTransientException =>
       throw new NormalShardException(e.toString, id, null)
     case e: SQLException =>
@@ -58,6 +66,10 @@ class ShardExceptionWrappingQueryEvaluator(shardId: ShardId, evaluator: QueryEva
         throw new ShardTimeoutException(e.timeout, shardId, e)
       case e: SqlDatabaseTimeoutException =>
         throw new ShardDatabaseTimeoutException(e.timeout, shardId, e)
+      case e: PoolTimeoutException =>
+        throw new NormalShardException(e.toString, shardId, e)
+      case e: PoolEmptyException =>
+        throw new NormalShardException(e.toString, shardId, e)
       case e: MySQLTransientException =>
         throw new NormalShardException(e.toString, shardId, null)
       case e: SQLException =>

src/main/scala/com/twitter/gizzard/shards/LeafRoutingNode.scala

@@ -51,9 +51,8 @@ class LeafRoutingNode[T](private[shards] val factory: ShardFactory[T], val shard
 
   val children = Nil
 
-  // only one of these will usually be called.
-  lazy val readOnlyShard  = factory.instantiateReadOnly(shardInfo, weight)
-  lazy val readWriteShard = factory.instantiate(shardInfo, weight)
+  val readOnlyShard  = factory.instantiateReadOnly(shardInfo, weight)
+  val readWriteShard = factory.instantiate(shardInfo, weight)
 
   override def shardInfos = Seq(shardInfo)
 

src/main/scala/com/twitter/gizzard/shards/RoutingNode.scala

@@ -36,7 +36,7 @@ abstract class RoutingNode[T] {
   def children: Seq[RoutingNode[T]]
   protected[shards] def collectedShards(readOnly: Boolean): Seq[Leaf[T]]
 
-  protected val log = Logger.get
+  protected val exceptionLog = Logger.get("exception")
 
   def shardInfos: Seq[ShardInfo] = children flatMap { _.shardInfos }
 
@@ -107,7 +107,7 @@ abstract class RoutingNode[T] {
   protected def logException(e: Throwable, shard: T, id: ShardId) {
     val normalized = normalizeException(e, id)
 
-    log.warning(normalized, "Error on %s: %s", id, e)
+    exceptionLog.warning(normalized, "Error on %s: %s", id, e)
   }
 
   protected def normalizeException(ex: Throwable, shardId: ShardId): Throwable = ex match {

src/test/scala/com/twitter/gizzard/nameserver/NameServerSpec.scala

@@ -32,15 +32,26 @@ object NameServerSpec extends ConfiguredSpecification with JMocker with ClassMoc
 
     val shard        = mock[AnyRef]
     var shardFactory = mock[ShardFactory[AnyRef]]
-    val nodes        = shardInfos map { new LeafRoutingNode(shardFactory, _, 1) }
-    val replNode     = ReplicatingShard(replicatingInfo, 1, Seq(nodes(3)))
+    var nodes: Seq[LeafRoutingNode[AnyRef]]        = null
+    var replNode: ReplicatingShard[AnyRef]         = null
 
     doBefore {
       expect {
         one(nameServerShard).reload()
         one(nameServerShard).currentState()    willReturn Seq(nameServerState)
+        2.of(shardFactory).instantiateReadOnly(shardInfos(0), 1) willReturn shard
+        2.of(shardFactory).instantiate(shardInfos(0), 1) willReturn shard
+        2.of(shardFactory).instantiateReadOnly(shardInfos(1), 1) willReturn shard
+        2.of(shardFactory).instantiate(shardInfos(1), 1) willReturn shard
+        2.of(shardFactory).instantiateReadOnly(shardInfos(2), 1) willReturn shard
+        2.of(shardFactory).instantiate(shardInfos(2), 1) willReturn shard
+        2.of(shardFactory).instantiateReadOnly(shardInfos(3), 1) willReturn shard
+        2.of(shardFactory).instantiate(shardInfos(3), 1) willReturn shard
       }
 
+      nodes        = shardInfos map { new LeafRoutingNode(shardFactory, _, 1) }
+      replNode     = ReplicatingShard(replicatingInfo, 1, Seq(nodes(3)))
+
       nameServer = new NameServer(LeafRoutingNode(nameServerShard), identity)
       forwarder  = nameServer.configureMultiForwarder[AnyRef](
         _.shardFactories(SQL_SHARD -> shardFactory)
@@ -87,8 +98,10 @@ object NameServerSpec extends ConfiguredSpecification with JMocker with ClassMoc
         one(nameServerShard).listDownwardLinks(replicatingInfo.id) willReturn linksList
         one(nameServerShard).getShard(shardInfos(3).id)            willReturn shardInfos(3)
         one(nameServerShard).listDownwardLinks(shardInfos(3).id)   willReturn List[LinkInfo]()
-        never(shardFactory).instantiate(shardInfos(2), 1)          willReturn shard
-        never(shardFactory).instantiate(shardInfos(3), 1)          willReturn shard
+        one(shardFactory).instantiateReadOnly(shardInfos(2), 1) willReturn shard
+        one(shardFactory).instantiate(shardInfos(2), 1) willReturn shard
+        one(shardFactory).instantiateReadOnly(shardInfos(3), 1) willReturn shard
+        one(shardFactory).instantiate(shardInfos(3), 1) willReturn shard
       }
 
       forwarder.findShardById(shardInfos(2).id)   mustEqual Some(nodes(2))
@@ -99,6 +112,8 @@ object NameServerSpec extends ConfiguredSpecification with JMocker with ClassMoc
       val floatingShard = ShardInfo(ShardId("localhost", "floating"), SQL_SHARD, "a", "b", Busy.Normal)
 
       expect {
+        2.of(shardFactory).instantiateReadOnly(floatingShard, 1)
+        2.of(shardFactory).instantiate(floatingShard, 1)
         one(nameServerShard).getShard(floatingShard.id)          willReturn floatingShard
         one(nameServerShard).listDownwardLinks(floatingShard.id) willReturn List[LinkInfo]()
       }