Permalink
Browse files

Merge branch 'master' into release_3_0

Conflicts:
	docs/guide.md
	project/Build.scala
	project/plugins.sbt
	src/main/scala/net/lag/kestrel/KestrelHandler.scala
  • Loading branch information...
2 parents fefd746 + eb52b41 commit 3efbe982931c218f674003c25273f853be092ff0 Robey Pointer committed May 30, 2012
Showing with 676 additions and 227 deletions.
  1. +17 −0 ChangeLog
  2. +1 −5 loadtest/.gitignore
  3. +2 −2 loadtest/README.md
  4. +16 −10 loadtest/config/{kestrel-dataservices-consumer.scala → kestrel-memcache-consumer.scala}
  5. +17 −10 loadtest/config/{kestrel-dataservices-producer.scala → kestrel-memcache-producer.scala}
  6. +47 −0 loadtest/config/kestrel-thrift-consumer.scala
  7. +46 −0 loadtest/config/kestrel-thrift-producer.scala
  8. +35 −0 loadtest/project/Build.scala
  9. +0 −9 loadtest/project/build.properties
  10. +0 −35 loadtest/project/build/LoadtestProject.scala
  11. +24 −0 loadtest/project/plugins.sbt
  12. +0 −23 loadtest/project/plugins/Plugins.scala
  13. +0 −46 loadtest/src/main/scala/net/lag/kestrel/loadtest/AbstractKestrelLoadTest.scala
  14. +0 −47 loadtest/src/main/scala/net/lag/kestrel/loadtest/KestrelConsumerLoadTest.scala
  15. +114 −0 loadtest/src/main/scala/net/lag/kestrel/loadtest/KestrelConsumerLoadTestConfig.scala
  16. +7 −0 loadtest/src/main/scala/net/lag/kestrel/loadtest/KestrelLoadTestConfig.scala
  17. +0 −26 loadtest/src/main/scala/net/lag/kestrel/loadtest/KestrelProducerLoadTest.scala
  18. +75 −0 loadtest/src/main/scala/net/lag/kestrel/loadtest/KestrelProducerLoadTestConfig.scala
  19. +21 −0 loadtest/src/main/scala/net/lag/kestrel/loadtest/KestrelRecordProcessor.scala
  20. +40 −0 loadtest/src/main/scala/net/lag/kestrel/loadtest/memcache/AbstractKestrelMemcacheLoadTest.scala
  21. +23 −0 loadtest/src/main/scala/net/lag/kestrel/loadtest/memcache/KestrelMemcacheConsumer.scala
  22. +19 −0 loadtest/src/main/scala/net/lag/kestrel/loadtest/memcache/KestrelMemcacheProducer.scala
  23. +29 −0 loadtest/src/main/scala/net/lag/kestrel/loadtest/thrift/AbstractKestrelThriftLoadTest.scala
  24. +35 −0 loadtest/src/main/scala/net/lag/kestrel/loadtest/thrift/KestrelThriftConsumer.scala
  25. +25 −0 loadtest/src/main/scala/net/lag/kestrel/loadtest/thrift/KestrelThriftProducer.scala
  26. +1 −0 loadtest/src/main/thrift/kestrel.thrift
  27. +2 −2 project/plugins.sbt
  28. +18 −11 src/main/scala/net/lag/kestrel/KestrelHandler.scala
  29. +2 −1 src/scripts/kestrel.sh
  30. +60 −0 src/test/scala/net/lag/kestrel/KestrelHandlerSpec.scala
View
@@ -1,3 +1,20 @@
+2.2.0
+-----
+release: 11 May 2012
+
+- Thrift interface. See src/main/thrift/kestrel.thrift. [Jason Jackson]
+- Kestrel now uses the Finagle library for networking.
+- Allow max items to be specified for memcache protocol monitor command
+- maxExpireSweep configuration to limit the number of expired items removed
+ from a queue per sweep [Hitoshi Asai]
+- maxQueueAge configuration auto deletes empty queues that have existed
+ longer than the specified time [Cory G Watson]
+- New statistics:
+ queue_creates, queue_deletes, queue_expires (global) [Cory G Watson]
+ cmd_monitor, cmd_monitor_get (global)
+ total_flushes, create_time (per-queue) [Cory G Watson]
+- Better error reporting on startup failure
+- Scala 2.9.1, sbt11, sbt-package-dist 1.0.5
2.1.5
-----
View
@@ -1,8 +1,4 @@
.ivyjars
.run_classpath
-project/boot/
-project/plugins/lib_managed/
-project/plugins/src_managed/
-project/plugins/project/
-lib_managed/
+target/
View
@@ -8,9 +8,9 @@ From the project root directory...
$ cd loadtest
$ sbt update package-dist # only necessary if libs change
- $ sbt start-parrot-consumer
+ $ sbt run-parrot\ config/kestrel-memcache-consumer.scala
...
- $ sbt start-parrot-producer
+ $ sbt run-parrot\ config/kestrel-memcache-producer.scala
...
Modify the config files in loadtest/config to choose victims,
@@ -4,15 +4,17 @@ import java.io._
new ParrotLauncherConfig {
mesosCluster = "smfd-devel"
hadoopNS = "hdfs://hadoop-scribe-nn.smfd.twitter.com"
+ hadoopConfig = "/etc/hadoop/hadoop-conf-smfd"
+
zkHostName = Some("zookeeper.smfd.twitter.com")
distDir = "dist/kestrel_loadtest"
- jobName = "kestrel_consumer"
+ jobName = "kestrel_memcache_consumer"
port = 22133
victims = "smfd-akc-04-sr1.devel.twitter.com"
parser = "thrift" // magic
- hostConnectionLimit = 2000
+ hostConnectionLimit = 5000
log = {
val file = File.createTempFile("kestrel", "parrot")
@@ -21,20 +23,24 @@ new ParrotLauncherConfig {
writer.close
file.getAbsolutePath
}
- requestRate = 100
+ requestRate = 1300
numInstances = 1
- duration = 10
+ duration = 75
timeUnit = "MINUTES"
- imports = """import net.lag.kestrel.loadtest.KestrelConsumerLoadTest
- import com.twitter.finagle.kestrel.protocol.Response"""
+ imports = """import net.lag.kestrel.loadtest.memcache.KestrelMemcacheConsumer
+ import net.lag.kestrel.loadtest._
+ import com.twitter.finagle.kestrel.protocol.Response
+ import com.twitter.parrot.util.SlowStartPoissonProcess
+ import com.twitter.conversions.time._"""
+
+ createDistribution = "createDistribution = { rate => new SlowStartPoissonProcess(rate, 5.minutes) }"
+
responseType = "Response"
transport = "KestrelTransport"
- loadTest = """new KestrelConsumerLoadTest(kestrelService) {
- numQueues = 10
- numFanouts = 5
+ loadTest = """new KestrelMemcacheConsumer(service.get) {
+ distribution = ConsumerQueueDistribution.simple("vshard_%d", 10, 10)
timeout = 100
- queueNameTemplate = "vshard_%d"
}"""
doConfirm = false
@@ -4,15 +4,17 @@ import java.io._
new ParrotLauncherConfig {
mesosCluster = "smfd-devel"
hadoopNS = "hdfs://hadoop-scribe-nn.smfd.twitter.com"
+ hadoopConfig = "/etc/hadoop/hadoop-conf-smfd"
+
zkHostName = Some("zookeeper.smfd.twitter.com")
distDir = "dist/kestrel_loadtest"
- jobName = "kestrel_producer"
+ jobName = "kestrel_memcache_producer"
port = 22133
victims = "smfd-akc-04-sr1.devel.twitter.com"
parser = "thrift" // magic
- hostConnectionLimit = 500
+ hostConnectionLimit = 5000
log = {
val file = File.createTempFile("kestrel", "parrot")
@@ -21,19 +23,24 @@ new ParrotLauncherConfig {
writer.close
file.getAbsolutePath
}
- requestRate = 100
+ requestRate = 1250
numInstances = 1
- duration = 10
+ duration = 60
timeUnit = "MINUTES"
- imports = """import net.lag.kestrel.loadtest.KestrelProducerLoadTest
- import com.twitter.finagle.kestrel.protocol.Response"""
+ imports = """import net.lag.kestrel.loadtest.memcache.KestrelMemcacheProducer
+ import net.lag.kestrel.loadtest._
+ import com.twitter.finagle.kestrel.protocol.Response
+ import com.twitter.parrot.util.SlowStartPoissonProcess
+ import com.twitter.conversions.time._
+ import com.twitter.conversions.storage._"""
+
+ createDistribution = "createDistribution = { rate => new SlowStartPoissonProcess(rate, 5.minutes) }"
+
responseType = "Response"
transport = "KestrelTransport"
- loadTest = """new KestrelProducerLoadTest(kestrelService) {
- numQueues = 10
- payloadSize = 50
- queueNameTemplate = "vshard_%d"
+ loadTest = """new KestrelMemcacheProducer(service.get) {
+ distribution = ProducerQueueDistribution.simple("vshard_%d", 10, 50.bytes)
}"""
doConfirm = false
@@ -0,0 +1,47 @@
+import com.twitter.parrot.config.ParrotLauncherConfig
+import java.io._
+
+new ParrotLauncherConfig {
+ mesosCluster = "smfd-devel"
+ hadoopNS = "hdfs://hadoop-scribe-nn.smfd.twitter.com"
+ hadoopConfig = "/etc/hadoop/hadoop-conf-smfd"
+
+ zkHostName = Some("zookeeper.smfd.twitter.com")
+
+ distDir = "dist/kestrel_loadtest"
+ jobName = "kestrel_thrift_consumer"
+ port = 2229
+ victims = "smfd-akc-04-sr1.devel.twitter.com"
+ parser = "thrift" // magic
+
+ hostConnectionLimit = 5000
+
+ log = {
+ val file = File.createTempFile("kestrel", "parrot")
+ val writer = new FileWriter(file)
+ (1 to 50000).foreach { x => writer.write("dummy %d\n".format(x)) }
+ writer.close
+ file.getAbsolutePath
+ }
+
+ requestRate = 1300
+ numInstances = 1
+ duration = 75
+ timeUnit = "MINUTES"
+
+ imports = """import net.lag.kestrel.loadtest.thrift.KestrelThriftConsumer
+ import net.lag.kestrel.loadtest._
+ import com.twitter.parrot.util.SlowStartPoissonProcess
+ import com.twitter.conversions.time._"""
+
+ createDistribution = "createDistribution = { rate => new SlowStartPoissonProcess(rate, 5.minutes) }"
+
+ responseType = "Array[Byte]"
+ transport = "ThriftTransport"
+ loadTest = """new KestrelThriftConsumer(service.get) {
+ distribution = ConsumerQueueDistribution.simple("vshard_%d", 10, 9)
+ timeout = 100
+ }"""
+
+ doConfirm = false
+}
@@ -0,0 +1,46 @@
+import com.twitter.parrot.config.ParrotLauncherConfig
+import java.io._
+
+new ParrotLauncherConfig {
+ mesosCluster = "smfd-devel"
+ hadoopNS = "hdfs://hadoop-scribe-nn.smfd.twitter.com"
+ hadoopConfig = "/etc/hadoop/hadoop-conf-smfd"
+
+ zkHostName = Some("zookeeper.smfd.twitter.com")
+
+ distDir = "dist/kestrel_loadtest"
+ jobName = "kestrel_thrift_producer"
+ port = 2229
+ victims = "smfd-akc-04-sr1.devel.twitter.com"
+ parser = "thrift" // magic
+
+ hostConnectionLimit = 5000
+
+ log = {
+ val file = File.createTempFile("kestrel", "parrot")
+ val writer = new FileWriter(file)
+ (1 to 50000).foreach { x => writer.write("dummy %d\n".format(x)) }
+ writer.close
+ file.getAbsolutePath
+ }
+ requestRate = 125
+ numInstances = 1
+ duration = 60
+ timeUnit = "MINUTES"
+
+ imports = """import net.lag.kestrel.loadtest.thrift.KestrelThriftProducer
+ import net.lag.kestrel.loadtest._
+ import com.twitter.parrot.util.SlowStartPoissonProcess
+ import com.twitter.conversions.time._
+ import com.twitter.conversions.storage._"""
+
+ createDistribution = "createDistribution = { rate => new SlowStartPoissonProcess(rate, 5.minutes) }"
+
+ responseType = "Array[Byte]"
+ transport = "ThriftTransport"
+ loadTest = """new KestrelThriftProducer(service.get) {
+ distribution = ProducerQueueDistribution.simple("vshard_%d", 10, 50.bytes)
+ }"""
+
+ doConfirm = false
+}
@@ -0,0 +1,35 @@
+import sbt._
+import Keys._
+import com.twitter.sbt._
+
+object Kestrel extends Build {
+ val parrotLauncherMain = "com.twitter.parrot.launcher.LauncherMain"
+ val runParrot = InputKey[Unit]("run-parrot")
+ val killParrot = InputKey[Unit]("kill-parrot")
+
+ lazy val root = Project(
+ id = "kestrel_loadtest",
+ base = file("."),
+ settings = Project.defaultSettings ++
+ StandardProject.newSettings ++
+ SubversionPublisher.newSettings ++
+ CompileThriftScrooge.newSettings
+ ).settings(
+ name := "kestrel_loadtest",
+ organization := "net.lag",
+ version := "1.0.0-SNAPSHOT",
+ scalaVersion := "2.8.1",
+
+ libraryDependencies ++= Seq(
+ "com.twitter" % "iago" % "0.4.7",
+ "com.twitter" %% "scrooge-runtime" % "1.1.3"
+ ),
+
+ CompileThriftScrooge.scroogeVersion := "1.1.7",
+ PackageDist.packageDistConfigFilesValidationRegex := None,
+ publishArtifact in Test := true,
+
+ fullRunInputTask(runParrot, Compile, parrotLauncherMain, "-f"),
+ fullRunInputTask(killParrot, Compile, parrotLauncherMain, "-k", "-f")
+ )
+}
@@ -1,9 +0,0 @@
-#Project properties
-#Thu Oct 13 10:08:00 PDT 2011
-project.organization=net.lag
-project.name=kestrel_loadtest
-sbt.version=0.7.4
-project.version=2.1.4-SNAPSHOT
-def.scala.version=2.7.7
-build.scala.versions=2.8.1
-project.initialize=false
@@ -1,35 +0,0 @@
-import sbt._
-import com.twitter.sbt._
-
-class LoadtestProject(info: ProjectInfo) extends StandardServiceProject(info)
- with ProjectDependencies
- with DefaultRepos
-{
- val twitterPrivate = "twitter.com" at "http://binaries.local.twitter.com/maven/"
-
- override def filterScalaJars = false
-
- override def mainClass = Some("com.twitter.parrot.launcher.LauncherMain")
-
- lazy val startParrotConsumer = runParrot("config/kestrel-dataservices-consumer.scala")
- lazy val killParrotConsumer = killParrot("config/kestrel-dataservices-consumer.scala")
-
- lazy val startParrotProducer = runParrot("config/kestrel-dataservices-producer.scala")
- lazy val killParrotProducer = killParrot("config/kestrel-dataservices-producer.scala")
-
- def runParrotTask(args: Array[String]) = task { _ =>
- runTask(mainClass, runClasspath, args) dependsOn(compile, copyResources)
- }
-
- def runParrot(config: String) = runParrotTask(Array("-f", config))
- def killParrot(config: String) = runParrotTask(Array("-f", config, "-k"))
-
- val parrot = "com.twitter" % "parrot" % "0.4.5"
-
- override def ivyXML =
- <dependencies>
- <exclude org="com.sun.jdmk"/>
- <exclude org="com.sun.jmx"/>
- <exclude org="javax.jms"/>
- </dependencies>
-}
@@ -0,0 +1,24 @@
+sbtResolver <<= (sbtResolver) { r =>
+ Option(System.getenv("SBT_PROXY_REPO")) map { x =>
+ Resolver.url("proxy repo for sbt", url(x))(Resolver.ivyStylePatterns)
+ } getOrElse r
+}
+
+resolvers <<= (resolvers) { r =>
+ (Option(System.getenv("SBT_PROXY_REPO")) map { url =>
+ Seq("proxy-repo" at url)
+ } getOrElse {
+ r ++ Seq(
+ "twitter.com" at "http://maven.twttr.com/",
+ "scala-tools" at "http://scala-tools.org/repo-releases/",
+ "maven" at "http://repo1.maven.org/maven2/",
+ "freemarker" at "http://freemarker.sourceforge.net/maven2/"
+ )
+ }) ++ Seq("local" at ("file:" + System.getProperty("user.home") + "/.m2/repository/"))
+}
+
+externalResolvers <<= (resolvers) map identity
+
+addSbtPlugin("com.twitter" %% "sbt-package-dist" % "1.0.5")
+
+addSbtPlugin("com.twitter" %% "sbt11-scrooge" % "1.0.0")
@@ -1,23 +0,0 @@
-import sbt._
-
-class Plugins(info: ProjectInfo) extends PluginDefinition(info) {
- import scala.collection.jcl
- val environment = jcl.Map(System.getenv())
- def isSBTOpenTwitter = environment.get("SBT_OPEN_TWITTER").isDefined
- def isSBTTwitter = environment.get("SBT_TWITTER").isDefined
-
- override def repositories = if (isSBTOpenTwitter) {
- Set("twitter.artifactory" at "http://artifactory.local.twitter.com/open-source/")
- } else if (isSBTTwitter) {
- Set("twitter.artifactory" at "http://artifactory.local.twitter.com/repo/")
- } else {
- super.repositories ++ Set(
- "twitter.com" at "http://maven.twttr.com/",
- "scala-tools" at "http://scala-tools.org/repo-releases/",
- "freemarker" at "http://freemarker.sourceforge.net/maven2/"
- )
- }
- override def ivyRepositories = Seq(Resolver.defaultLocal(None)) ++ repositories
-
- val standardProject = "com.twitter" % "standard-project" % "1.0.0"
-}
Oops, something went wrong.

0 comments on commit 3efbe98

Please sign in to comment.