Permalink
Browse files

added configuration options

  • Loading branch information...
1 parent 08a75c4 commit feaa4fddc4aaf2a60e9ede836e06012c8992193f Jason Jackson committed Sep 22, 2011
View
@@ -2,6 +2,91 @@ import com.twitter.conversions.time._
import com.twitter.logging.config._
import com.twitter.ostrich.admin.config._
import com.twitter.kestrelthrift.config._
+import com.twitter.conversions.storage._
+import com.twitter.conversions.time._
+import com.twitter.logging.config._
+import com.twitter.ostrich.admin.config._
+import net.lag.kestrel.config._
+
+new KestrelConfig {
+ listenAddress = "0.0.0.0"
+ memcacheListenPort = 22133
+ textListenPort = 2222
+
+ queuePath = "./data"
+
+ clientTimeout = 30.seconds
+
+ expirationTimerFrequency = 1.second
+
+ maxOpenTransactions = 100
+
+ // default queue settings:
+ default.defaultJournalSize = 16.megabytes
+ default.maxMemorySize = 128.megabytes
+ default.maxJournalSize = 1.gigabyte
+
+ admin.httpPort = 2223
+
+ admin.statsNodes = new StatsConfig {
+ reporters = new JsonStatsLoggerConfig {
+ loggerName = "stats"
+ serviceName = "kestrel"
+ } :: new TimeSeriesCollectorConfig
+ }
+
+ queues = new QueueBuilder {
+ // keep items for no longer than a half hour, and don't accept any more if
+ // the queue reaches 1.5M items.
+ name = "weather_updates"
+ maxAge = 1800.seconds
+ maxItems = 1500000
+ } :: new QueueBuilder {
+ // don't keep a journal file for this queue. when kestrel exits, any
+ // remaining contents will be lost.
+ name = "transient_events"
+ keepJournal = false
+ } :: new QueueBuilder {
+ name = "jobs_pending"
+ expireToQueue = "jobs_ready"
+ maxAge = 30.seconds
+ } :: new QueueBuilder {
+ name = "jobs_ready"
+ syncJournal = 0.seconds
+ } :: new QueueBuilder {
+ name = "spam"
+ } :: new QueueBuilder {
+ name = "spam0"
+ } :: new QueueBuilder {
+ name = "hello"
+ fanoutOnly = true
+ } :: new QueueBuilder {
+ name = "small"
+ maxSize = 128.megabytes
+ maxMemorySize = 16.megabytes
+ maxJournalSize = 128.megabytes
+ discardOldWhenFull = true
+ } :: new QueueBuilder {
+ name = "slow"
+ syncJournal = 10.milliseconds
+ }
+
+ loggers = new LoggerConfig {
+ level = Level.INFO
+ handlers = new FileHandlerConfig {
+ filename = "/var/log/kestrel/kestrel.log"
+ roll = Policy.Never
+ }
+ } :: new LoggerConfig {
+ node = "stats"
+ level = Level.INFO
+ useParents = false
+ handlers = new FileHandlerConfig {
+ filename = "/var/log/kestrel/stats.log"
+ roll = Policy.Never
+ }
+ }
+}
// development mode.
new KestrelthriftServiceConfig {
View
@@ -2,6 +2,69 @@ import com.twitter.conversions.time._
import com.twitter.logging.config._
import com.twitter.ostrich.admin.config._
import com.twitter.kestrelthrift.config._
+import com.twitter.conversions.storage._
+import com.twitter.ostrich.admin.config._
+import net.lag.kestrel.config._
+
+new KestrelConfig {
+ listenAddress = "0.0.0.0"
+ memcacheListenPort = 22133
+ textListenPort = 2222
+
+ queuePath = "/var/spool/kestrel"
+
+ clientTimeout = None
+
+ expirationTimerFrequency = 1.second
+
+ maxOpenTransactions = 100
+
+ // default queue settings:
+ default.defaultJournalSize = 16.megabytes
+ default.maxMemorySize = 128.megabytes
+ default.maxJournalSize = 1.gigabyte
+ default.syncJournal = 20.milliseconds
+
+ admin.httpPort = 2223
+
+ admin.statsNodes = new StatsConfig {
+ reporters = new JsonStatsLoggerConfig {
+ loggerName = "stats"
+ serviceName = "kestrel"
+ } :: new TimeSeriesCollectorConfig
+ }
+
+ queues = new QueueBuilder {
+ // keep items for no longer than a half hour, and don't accept any more if
+ // the queue reaches 1.5M items.
+ name = "weather_updates"
+ maxAge = 1800.seconds
+ maxItems = 1500000
+ } :: new QueueBuilder {
+ // don't keep a journal file for this queue. when kestrel exits, any
+ // remaining contents will be lost.
+ name = "transient_events"
+ keepJournal = false
+ }
+
+ loggers = new LoggerConfig {
+ level = Level.INFO
+ handlers = new FileHandlerConfig {
+ filename = "/var/log/kestrel/kestrel.log"
+ roll = Policy.SigHup
+ }
+ } :: new LoggerConfig {
+ node = "stats"
+ level = Level.INFO
+ useParents = false
+ handlers = new ScribeHandlerConfig {
+ hostname = "localhost"
+ category = "cuckoo_json"
+ maxMessagesPerTransaction = 100
+ formatter = BareFormatterConfig
+ }
+ }
+}
// production mode.
new KestrelthriftServiceConfig {
@@ -7,17 +7,17 @@ import config._
import net.lag.kestrel._
import net.lag.kestrel.config._
import java.nio.ByteBuffer
+import java.util.concurrent.{TimeUnit}
+import org.jboss.netty.util.{HashedWheelTimer, Timeout, Timer, TimerTask}
class KestrelthriftServiceImpl(config: KestrelthriftServiceConfig) extends KestrelthriftServiceServer {
val serverName = "Kestrelthrift"
val thriftPort = config.thriftPort
- //val dataDir = "/var/spool/kestrel"
- val dataDir = "./data" // TODO: move to config
- // TODO: commit
- // TODO: batch get
-
- val qs = new QueueCollection(dataDir, new FakeTimer(), new QueueBuilder().apply(), List())
+ // this means no timeout will be at better granularity than 10ms.
+ val timer = new HashedWheelTimer(10, TimeUnit.MILLISECONDS)
+ val qs = new QueueCollection("<ignored>", new NettyTimer(timer), new QueueBuilder().apply(), List())
+ qs.loadQueues()
def get(queueName: String, maxItems: Int, transaction: Boolean) = {
val futureList = for(i <- 1 to maxItems)

0 comments on commit feaa4fd

Please sign in to comment.