This repository has been archived by the owner on Sep 18, 2021. It is now read-only.
forked from nkallen/querulous
-
Notifications
You must be signed in to change notification settings - Fork 39
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into improved_stats
Conflicts: project/build.properties
- Loading branch information
Showing
29 changed files
with
647 additions
and
111 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,8 +1,8 @@ | ||
| #Project properties | ||
| #Thu May 19 13:06:29 PDT 2011 | ||
| #Mon Jun 06 13:52:29 PDT 2011 | ||
| project.organization=com.twitter | ||
| project.name=querulous | ||
| sbt.version=0.7.4 | ||
| project.version=2.1.6-stats-alpha1-SNAPSHOT | ||
| project.version=2.2.1-stats-alpha2-SNAPSHOT | ||
| build.scala.versions=2.8.1 | ||
| project.initialize=false |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| #Automatically generated by ReleaseManagement | ||
| #Mon Jun 06 13:52:29 PDT 2011 | ||
| version=2.2.0 | ||
| sha1=c8773aac4ff6f4890d8b7dbb59fd68ea60d53118 |
22 changes: 22 additions & 0 deletions
22
src/main/scala/com/twitter/querulous/DaemonThreadFactory.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| package com.twitter.querulous | ||
|
|
||
| import java.util.concurrent.{ThreadFactory, TimeoutException => JTimeoutException, _} | ||
| import java.util.concurrent.atomic.AtomicInteger | ||
| import com.twitter.util.Duration | ||
|
|
||
|
|
||
| class DaemonThreadFactory extends ThreadFactory { | ||
| val group = new ThreadGroup(Thread.currentThread().getThreadGroup(), "querulous") | ||
| val threadNumber = new AtomicInteger(1) | ||
|
|
||
| def newThread(r: Runnable) = { | ||
| val thread = new Thread(group, r, "querulous-" + threadNumber.getAndIncrement()) | ||
| if (!thread.isDaemon) { | ||
| thread.setDaemon(true) | ||
| } | ||
| if (thread.getPriority != Thread.NORM_PRIORITY) { | ||
| thread.setPriority(Thread.NORM_PRIORITY) | ||
| } | ||
| thread | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
27 changes: 27 additions & 0 deletions
27
src/main/scala/com/twitter/querulous/async/AsyncDatabase.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| package com.twitter.querulous.async | ||
|
|
||
| import java.sql.Connection | ||
| import com.twitter.util.Future | ||
|
|
||
|
|
||
| trait AsyncDatabaseFactory { | ||
| def apply( | ||
| hosts: List[String], | ||
| name: String, | ||
| username: String, | ||
| password: String, | ||
| urlOptions: Map[String, String] | ||
| ): AsyncDatabase | ||
|
|
||
| def apply(hosts: List[String], name: String, username: String, password: String): AsyncDatabase = { | ||
| apply(hosts, name, username, password, Map.empty) | ||
| } | ||
|
|
||
| def apply(hosts: List[String], username: String, password: String): AsyncDatabase = { | ||
| apply(hosts, null, username, password, Map.empty) | ||
| } | ||
| } | ||
|
|
||
| trait AsyncDatabase { | ||
| def withConnection[R](f: Connection => R): Future[R] | ||
| } |
112 changes: 112 additions & 0 deletions
112
src/main/scala/com/twitter/querulous/async/AsyncQueryEvaluator.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| package com.twitter.querulous.async | ||
|
|
||
| import java.util.concurrent.Executors | ||
| import java.sql.ResultSet | ||
| import com.twitter.util.{Future, FuturePool} | ||
| import com.twitter.querulous.config.{Connection => ConnectionConfig} | ||
| import com.twitter.querulous.DaemonThreadFactory | ||
| import com.twitter.querulous.evaluator._ | ||
| import com.twitter.querulous.query.{QueryClass, SqlQueryFactory} | ||
| import com.twitter.querulous.database.ThrottledPoolingDatabaseFactory | ||
| import com.twitter.conversions.time._ | ||
|
|
||
|
|
||
| object AsyncQueryEvaluator extends AsyncQueryEvaluatorFactory { | ||
| lazy val defaultFuturePool = FuturePool(Executors.newCachedThreadPool(new DaemonThreadFactory)) | ||
|
|
||
| private def createEvaluatorFactory() = { | ||
| new StandardAsyncQueryEvaluatorFactory( | ||
| new BlockingDatabaseWrapperFactory( | ||
| defaultFuturePool, | ||
| new ThrottledPoolingDatabaseFactory(10, 100.millis, 10.seconds, 1.second) | ||
| ), | ||
| new SqlQueryFactory | ||
| ) | ||
| } | ||
|
|
||
| def apply( | ||
| dbhosts: List[String], | ||
| dbname: String, | ||
| username: String, | ||
| password: String, | ||
| urlOptions: Map[String, String] | ||
| ): AsyncQueryEvaluator = { | ||
| createEvaluatorFactory()(dbhosts, dbname, username, password, urlOptions) | ||
| } | ||
| } | ||
|
|
||
| trait AsyncQueryEvaluatorFactory { | ||
| def apply( | ||
| dbhosts: List[String], | ||
| dbname: String, | ||
| username: String, | ||
| password: String, | ||
| urlOptions: Map[String, String] | ||
| ): AsyncQueryEvaluator | ||
|
|
||
| def apply(dbhost: String, dbname: String, username: String, password: String, urlOptions: Map[String, String]): AsyncQueryEvaluator = { | ||
| apply(List(dbhost), dbname, username, password, urlOptions) | ||
| } | ||
|
|
||
| def apply(dbhosts: List[String], dbname: String, username: String, password: String): AsyncQueryEvaluator = { | ||
| apply(dbhosts, dbname, username, password, Map[String,String]()) | ||
| } | ||
|
|
||
| def apply(dbhost: String, dbname: String, username: String, password: String): AsyncQueryEvaluator = { | ||
| apply(List(dbhost), dbname, username, password, Map[String,String]()) | ||
| } | ||
|
|
||
| def apply(dbhost: String, username: String, password: String): AsyncQueryEvaluator = { | ||
| apply(List(dbhost), null, username, password, Map[String,String]()) | ||
| } | ||
|
|
||
| def apply(dbhosts: List[String], username: String, password: String): AsyncQueryEvaluator = { | ||
| apply(dbhosts, null, username, password, Map[String,String]()) | ||
| } | ||
|
|
||
| def apply(connection: ConnectionConfig): AsyncQueryEvaluator = { | ||
| apply(connection.hostnames.toList, connection.database, connection.username, connection.password, connection.urlOptions) | ||
| } | ||
| } | ||
|
|
||
| trait AsyncQueryEvaluator { | ||
| def select[A](queryClass: QueryClass, query: String, params: Any*)(f: ResultSet => A): Future[Seq[A]] | ||
|
|
||
| def select[A](query: String, params: Any*)(f: ResultSet => A): Future[Seq[A]] = { | ||
| select(QueryClass.Select, query, params: _*)(f) | ||
| } | ||
|
|
||
| def selectOne[A](queryClass: QueryClass, query: String, params: Any*)(f: ResultSet => A): Future[Option[A]] | ||
|
|
||
| def selectOne[A](query: String, params: Any*)(f: ResultSet => A): Future[Option[A]] = { | ||
| selectOne(QueryClass.Select, query, params: _*)(f) | ||
| } | ||
|
|
||
| def count(queryClass: QueryClass, query: String, params: Any*): Future[Int] | ||
|
|
||
| def count(query: String, params: Any*): Future[Int] = { | ||
| count(QueryClass.Select, query, params: _*) | ||
| } | ||
|
|
||
| def execute(queryClass: QueryClass, query: String, params: Any*): Future[Int] | ||
|
|
||
| def execute(query: String, params: Any*): Future[Int] = { | ||
| execute(QueryClass.Execute, query, params: _*) | ||
| } | ||
|
|
||
| def executeBatch(queryClass: QueryClass, query: String)(f: ParamsApplier => Unit): Future[Int] | ||
|
|
||
| def executeBatch(query: String)(f: ParamsApplier => Unit): Future[Int] = { | ||
| executeBatch(QueryClass.Execute, query)(f) | ||
| } | ||
|
|
||
| def nextId(tableName: String): Future[Long] | ||
|
|
||
| def insert(queryClass: QueryClass, query: String, params: Any*): Future[Long] | ||
|
|
||
| def insert(query: String, params: Any*): Future[Long] = { | ||
| insert(QueryClass.Execute, query, params: _*) | ||
| } | ||
|
|
||
| def transaction[T](f: Transaction => T): Future[T] | ||
| } |
76 changes: 76 additions & 0 deletions
76
src/main/scala/com/twitter/querulous/async/BlockingDatabaseWrapper.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| package com.twitter.querulous.async | ||
|
|
||
| import java.util.concurrent.Executors | ||
| import java.sql.Connection | ||
| import com.twitter.util.{Return, Throw, Future, Promise, FuturePool, JavaTimer, TimeoutException} | ||
| import com.twitter.querulous.DaemonThreadFactory | ||
| import com.twitter.querulous.database.{Database, DatabaseFactory} | ||
|
|
||
|
|
||
| class BlockingDatabaseWrapperFactory(pool: => FuturePool, factory: DatabaseFactory) | ||
| extends AsyncDatabaseFactory { | ||
| def apply( | ||
| hosts: List[String], | ||
| name: String, | ||
| username: String, | ||
| password: String, | ||
| urlOptions: Map[String, String] | ||
| ): AsyncDatabase = { | ||
| new BlockingDatabaseWrapper( | ||
| pool, | ||
| factory(hosts, name, username, password, urlOptions) | ||
| ) | ||
| } | ||
| } | ||
|
|
||
| private object AsyncConnectionCheckout { | ||
| lazy val checkoutTimer = new JavaTimer(true) | ||
| } | ||
|
|
||
| class BlockingDatabaseWrapper( | ||
| pool: FuturePool, | ||
| protected[async] val database: Database) | ||
| extends AsyncDatabase { | ||
|
|
||
| import AsyncConnectionCheckout._ | ||
|
|
||
| // XXX: this probably should be configurable as well. | ||
| private val checkoutPool = FuturePool(Executors.newSingleThreadExecutor(new DaemonThreadFactory)) | ||
| private val openTimeout = database.openTimeout | ||
|
|
||
| def withConnection[R](f: Connection => R) = { | ||
| checkoutConnection() flatMap { conn => | ||
| pool { | ||
| try { | ||
| f(conn) | ||
| } finally { | ||
| database.close(conn) | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private def checkoutConnection(): Future[Connection] = { | ||
| val promise = new Promise[Connection] | ||
|
|
||
| checkoutPool(database.open()) respond { rv => | ||
| // if the promise has already timed out, we need to close the connection here. | ||
| if (!promise.updateIfEmpty(rv)) rv foreach database.close | ||
| } | ||
|
|
||
| checkoutTimer.schedule(openTimeout.fromNow) { | ||
| promise.updateIfEmpty(Throw(new TimeoutException(openTimeout.toString))) | ||
| } | ||
|
|
||
| promise | ||
| } | ||
|
|
||
| // equality overrides | ||
|
|
||
| override def equals(other: Any) = other match { | ||
| case other: BlockingDatabaseWrapper => database eq other.database | ||
| case _ => false | ||
| } | ||
|
|
||
| override def hashCode = database.hashCode | ||
| } |
Oops, something went wrong.