Skip to content
Browse files

fix configuration in main

  • Loading branch information...
1 parent dcf4983 commit b61486b43792808910aa3159538fbdca54ca16ef Nick Kallen committed Apr 5, 2010
View
0 bad_jobs.log
No changes.
View
1 build.xml
@@ -10,6 +10,7 @@
<import file="${basedir}/ant/bootstrap.xml" />
<property name="install.dir" location="/usr/local/rowz" />
+ <property name="skip.docs" value="true" />
<target name="install" depends="package">
<mkdir dir="${install.dir}" />
<copy todir="${install.dir}">
View
156 config/development.conf
@@ -1,6 +1,156 @@
+admin_port = 9991
+admin_http_port = 9990
+
+host {
+ id = 1
+}
+
+db_default {
+ username = "$(DB_USERNAME)"
+ password = "$(DB_PASSWORD)"
+}
+
+w3c_header_period = 60000
+
+errors {
+ deadlock_retries = 3
+}
+
log {
+ filename = "test.log"
level = "info"
- console = false
- filename = "rowz.log"
- roll = "never"
+ roll = "hourly"
+ truncate_stack_traces = 100
+
+ w3c {
+ node = "w3c"
+ use_parents = false
+ filename = "w3c.log"
+ level = "info"
+ roll = "hourly"
+ }
+
+ bad_jobs {
+ node = "bad_jobs"
+ use_parents = false
+ filename = "bad_jobs.log"
+ level = "info"
+ roll = "never"
+ }
+}
+
+throttled_log {
+ period_msec = 60000
+ rate = 10
}
+
+rowz {
+ timeout_msec = 100
+ server_port = 7919
+ shard_server_port = 7920
+ job_server_port = 7921
+ min_threads = 200
+
+ w3c = [
+ "action-timing",
+ "db-timing",
+ "connection-pool-release-timing",
+ "connection-pool-reserve-timing",
+ "kestrel-put-timing",
+ "db-select-count",
+ "db-execute-count",
+ "job-success-count",
+ "operation",
+ "arguments"
+ ]
+
+ aggregate_jobs_page_size = 500
+
+ db (inherit="db_default") {
+ name = "rowz_data"
+
+ connection_pool {
+ max_wait = 20
+ min_evictable_idle_msec = -1
+ size_max = 10
+ size_min = 5
+ test_idle_msec = 1000
+ test_on_borrow = false
+
+ timeout {
+ initialize = 1000
+ open = 100
+ pool_size = 10
+ queue_size = 10000
+ }
+ }
+
+ disable {
+ error_count = 3
+ seconds = 60
+ }
+
+ query_timeout_default = 2000
+ queries {
+ }
+ }
+
+ nameserver (inherit="db_default") {
+ hostnames = ["localhost"]
+ name = "rowz_nameserver"
+
+ connection_pool {
+ max_wait = 20
+ min_evictable_idle_msec = -1
+ size_max = 1
+ size_min = 1
+ test_idle_msec = 1000
+ test_on_borrow = false
+
+ timeout {
+ initialize = 1000
+ open = 100
+ pool_size = 10
+ queue_size = 10000
+ }
+ }
+ query_timeout_default = 15000
+ queries { }
+ }
+
+ replication {
+ future {
+ pool_size = 100
+ max_pool_size = 100
+ keep_alive_time_seconds = 5
+ timeout_seconds = 6
+ }
+ }
+
+ queue {
+ path = "/tmp"
+ journal off
+ high {
+ job_queue = "high"
+ error_queue = "high_errors"
+ threads = 1
+ error_limit = 25
+ replay_interval = 900
+ }
+ medium {
+ job_queue = "medium"
+ error_queue = "medium_errors"
+ threads = 1
+ error_limit = 25
+ replay_interval = 900
+ }
+ low {
+ job_queue = "low"
+ error_queue = "low_errors"
+ threads = 1
+ error_limit = 25
+ replay_interval = 900
+ }
+ max_memory_size = 36000000
+ }
+}
View
153 config/production.conf
@@ -1,5 +1,156 @@
+admin_port = 9991
+admin_http_port = 9990
+
+host {
+ id = 1
+}
+
+db_default {
+ username = "$(DB_USERNAME)"
+ password = "$(DB_PASSWORD)"
+}
+
+w3c_header_period = 60000
+
+errors {
+ deadlock_retries = 3
+}
+
log {
- filename = "/var/log/rowz/production.log"
+ filename = "test.log"
level = "info"
roll = "hourly"
+ truncate_stack_traces = 100
+
+ w3c {
+ node = "w3c"
+ use_parents = false
+ filename = "w3c.log"
+ level = "info"
+ roll = "hourly"
+ }
+
+ bad_jobs {
+ node = "bad_jobs"
+ use_parents = false
+ filename = "bad_jobs.log"
+ level = "info"
+ roll = "never"
+ }
+}
+
+throttled_log {
+ period_msec = 60000
+ rate = 10
}
+
+rowz {
+ timeout_msec = 100
+ server_port = 7919
+ shard_server_port = 7920
+ job_server_port = 7921
+ min_threads = 200
+
+ w3c = [
+ "action-timing",
+ "db-timing",
+ "connection-pool-release-timing",
+ "connection-pool-reserve-timing",
+ "kestrel-put-timing",
+ "db-select-count",
+ "db-execute-count",
+ "job-success-count",
+ "operation",
+ "arguments"
+ ]
+
+ aggregate_jobs_page_size = 500
+
+ db (inherit="db_default") {
+ name = "rowz_data"
+
+ connection_pool {
+ max_wait = 20
+ min_evictable_idle_msec = -1
+ size_max = 10
+ size_min = 5
+ test_idle_msec = 1000
+ test_on_borrow = false
+
+ timeout {
+ initialize = 1000
+ open = 100
+ pool_size = 10
+ queue_size = 10000
+ }
+ }
+
+ disable {
+ error_count = 3
+ seconds = 60
+ }
+
+ query_timeout_default = 2000
+ queries {
+ }
+ }
+
+ nameserver (inherit="db_default") {
+ hostnames = ["localhost"]
+ name = "rowz_nameserver"
+
+ connection_pool {
+ max_wait = 20
+ min_evictable_idle_msec = -1
+ size_max = 1
+ size_min = 1
+ test_idle_msec = 1000
+ test_on_borrow = false
+
+ timeout {
+ initialize = 1000
+ open = 100
+ pool_size = 10
+ queue_size = 10000
+ }
+ }
+ query_timeout_default = 15000
+ queries { }
+ }
+
+ replication {
+ future {
+ pool_size = 100
+ max_pool_size = 100
+ keep_alive_time_seconds = 5
+ timeout_seconds = 6
+ }
+ }
+
+ queue {
+ path = "/tmp"
+ journal off
+ high {
+ job_queue = "high"
+ error_queue = "high_errors"
+ threads = 1
+ error_limit = 25
+ replay_interval = 900
+ }
+ medium {
+ job_queue = "medium"
+ error_queue = "medium_errors"
+ threads = 1
+ error_limit = 25
+ replay_interval = 900
+ }
+ low {
+ job_queue = "low"
+ error_queue = "low_errors"
+ threads = 1
+ error_limit = 25
+ replay_interval = 900
+ }
+ max_memory_size = 36000000
+ }
+}
View
24 config/test.conf
@@ -17,8 +17,26 @@ errors {
}
log {
- level = "error"
- console = true
+ filename = "test.log"
+ level = "info"
+ roll = "hourly"
+ truncate_stack_traces = 100
+
+ w3c {
+ node = "w3c"
+ use_parents = false
+ filename = "w3c.log"
+ level = "info"
+ roll = "hourly"
+ }
+
+ bad_jobs {
+ node = "bad_jobs"
+ use_parents = false
+ filename = "bad_jobs.log"
+ level = "info"
+ roll = "never"
+ }
}
throttled_log {
@@ -27,9 +45,11 @@ throttled_log {
}
rowz {
+ timeout_msec = 100
server_port = 7919
shard_server_port = 7920
job_server_port = 7921
+ min_threads = 200
w3c = [
"action-timing",
View
46 src/main/scala/com/twitter/rowz/Main.scala
@@ -1,7 +1,7 @@
package com.twitter.rowz
import com.twitter.gizzard.thrift.{JobManagerService, ShardManagerService}
-import net.lag.configgy.Configgy
+import net.lag.configgy.{Configgy, RuntimeEnvironment, ConfigMap}
import net.lag.logging.Logger
import com.twitter.gizzard.jobs.CopyFactory
import com.twitter.gizzard.nameserver.NameServer
@@ -20,27 +20,36 @@ object Main {
var jobServer: TSelectorServer = null
var shardServer: TSelectorServer = null
- val config = Configgy.config
- val w3c = new W3CStats(Logger.get("w3c"), config.getList("rowz.w3c").toArray)
+ var config: ConfigMap = null
+ val runtime = new RuntimeEnvironment(getClass)
def main(args: Array[String]) {
- state = Rowz(config, w3c)
- state.start()
- startThrift()
+ try {
+ runtime.load(args)
+ config = Configgy.config
+ val w3c = new W3CStats(Logger.get("w3c"), config.getList("rowz.w3c").toArray)
+ state = Rowz(config, w3c)
+ state.start()
+ startThrift(w3c)
+ } catch {
+ case _ =>
+ println("Exception in initialization!")
+ shutdown()
+ }
}
- def startThrift() {
- val timeout = config("timeout").toInt.milliseconds
- val executor = TSelectorServer.makeThreadPoolExecutor(config)
+ def startThrift(w3c: W3CStats) {
+ val timeout = config("rowz.timeout_msec").toInt.milliseconds
+ val executor = TSelectorServer.makeThreadPoolExecutor(config.configMap("rowz"))
val processor = new rowz.thrift.Rowz.Processor(LoggingProxy[rowz.thrift.Rowz.Iface](Stats, w3c, "Rowz", state.rowzService))
- rowzServer = TSelectorServer("rowz", config("port").toInt, processor, executor, timeout)
+ rowzServer = TSelectorServer("rowz", config("rowz.server_port").toInt, processor, executor, timeout)
val jobManagerService = new JobManagerService(state.prioritizingScheduler)
- val jobProcessor = new JobManager.Processor(LoggingProxy[JobManager.Iface](Stats, Main.w3c, "RowzJobs", jobManagerService))
+ val jobProcessor = new JobManager.Processor(LoggingProxy[JobManager.Iface](Stats, w3c, "RowzJobs", jobManagerService))
jobServer = TSelectorServer("rowz-jobs", config("rowz.job_server_port").toInt, jobProcessor, executor, timeout)
val shardManagerService = new ShardManagerService(state.nameServer, state.copyFactory, state.prioritizingScheduler(Priority.Medium.id))
- val shardProcessor = new ShardManager.Processor(ExceptionWrappingProxy(LoggingProxy[ShardManager.Iface](Stats, Main.w3c, "RowzShards", shardManagerService)))
+ val shardProcessor = new ShardManager.Processor(ExceptionWrappingProxy(LoggingProxy[ShardManager.Iface](Stats, w3c, "RowzShards", shardManagerService)))
shardServer = TSelectorServer("rowz-shards", config("rowz.shard_server_port").toInt, shardProcessor, executor, timeout)
rowzServer.serve()
@@ -49,10 +58,13 @@ object Main {
}
def shutdown() {
- rowzServer.stop()
- jobServer.stop()
- state.shutdown()
-
- System.exit(0)
+ try {
+ rowzServer.stop()
+ jobServer.stop()
+ state.shutdown()
+ } finally {
+ println("Exiting!")
+ System.exit(0)
+ }
}
}
View
6 src/main/scala/com/twitter/rowz/Rowz.scala
@@ -1,6 +1,6 @@
package com.twitter.rowz
-import net.lag.configgy.Config
+import net.lag.configgy.ConfigMap
import com.twitter.querulous.database.{ApachePoolingDatabaseFactory, MemoizingDatabaseFactory, DatabaseFactory}
import com.twitter.querulous.query.SqlQueryFactory
import com.twitter.querulous.evaluator.StandardQueryEvaluatorFactory
@@ -31,7 +31,7 @@ object Rowz {
def shutdown() = prioritizingScheduler.shutdown()
}
- def apply(config: Config, w3c: W3CStats): State = apply(
+ def apply(config: ConfigMap, w3c: W3CStats): State = apply(
config, w3c,
new MemoizingDatabaseFactory(new ApachePoolingDatabaseFactory(
config("rowz.db.connection_pool.size_min").toInt,
@@ -42,7 +42,7 @@ object Rowz {
config("rowz.db.connection_pool.min_evictable_idle_msec").toLong.millis))
)
- def apply(config: Config, w3c: W3CStats, databaseFactory: DatabaseFactory): State = {
+ def apply(config: ConfigMap, w3c: W3CStats, databaseFactory: DatabaseFactory): State = {
val queryEvaluatorFactory = new StandardQueryEvaluatorFactory(databaseFactory, new SqlQueryFactory)
val throttledLogger = new ThrottledLogger[String](Logger(), config("throttled_log.period_msec").toInt, config("throttled_log.rate").toInt)
View
834 test.log
834 additions, 0 deletions not shown because the diff is too large. Please use a local Git client to view these changes.
View
0 w3c.log
No changes.

0 comments on commit b61486b

Please sign in to comment.
Something went wrong with that request. Please try again.