Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

Add a couple new stats… #86

Merged
merged 1 commit into from

2 participants

@gphat

We create and destroy a lot of queues, so these stats would be useful to us.

While I was at it I thought flushes would be useful to add.

Global:
* queue_creates (total number of queues created)
* queue_deletes (total number of queues deleted)
Per-Queue:
* total_flushes (total number of times this queue has been flushed)

@gphat gphat Add more statistics along with tests and docs.
Global:
    * queue_creates (total number of queues created)
    * queue_deletes (total number of queues deleted)
Per-Queue:
    * total_flushes (total number of times this queue has been flushed)
0d8a507
@robey
Collaborator

thanks! :)

@robey robey merged commit 97ccc0c into twitter:master
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Feb 8, 2012
  1. @gphat

    Add more statistics along with tests and docs.

    gphat authored
    Global:
        * queue_creates (total number of queues created)
        * queue_deletes (total number of queues deleted)
    Per-Queue:
        * total_flushes (total number of times this queue has been flushed)
This page is out of date. Refresh to see the latest.
View
3  docs/guide.md
@@ -355,6 +355,8 @@ Global stats reported by kestrel are:
- `get_misses` - total `GET` requests on an empty queue
- `bytes_read` - total bytes read from clients
- `bytes_written` - total bytes written to clients
+- `queue_creates` - total number of queues created
+- `queue_deletes` - total number of queues deleted
For each queue, the following stats are also reported:
@@ -376,6 +378,7 @@ For each queue, the following stats are also reported:
- `waiters` - number of clients waiting for an item from this queue (using
`GET/t`)
- `open_transactions` - items read with `/open` but not yet confirmed
+- `total_flushes` total number of times this queue has been flushed
Kestrel as a library
View
2  src/main/scala/net/lag/kestrel/MemcacheHandler.scala
@@ -199,6 +199,8 @@ extends NettyHandler[MemcacheRequest](channelGroup, queueCollection, maxOpenTran
report += (("get_misses", Stats.getCounter("get_misses")().toString))
report += (("bytes_read", Stats.getCounter("bytes_read")().toString))
report += (("bytes_written", Stats.getCounter("bytes_written")().toString))
+ report += (("queue_creates", Stats.getCounter("queue_creates")().toString))
+ report += (("queue_deletes", Stats.getCounter("queue_deletes")().toString))
for (qName <- queues.queueNames) {
report ++= queues.stats(qName).map { case (k, v) => ("queue_" + qName + "_" + k, v) }
View
9 src/main/scala/net/lag/kestrel/PersistentQueue.scala
@@ -59,6 +59,10 @@ class PersistentQueue(val name: String, persistencePath: String, @volatile var c
val totalDiscarded = Stats.getCounter(statNamed("discarded"))
totalDiscarded.reset()
+ // # of times this queue has been flushed:
+ val totalFlushes = Stats.getCounter(statNamed("total_flushes"))
+ totalFlushes.reset()
+
// # of items in the queue (including those not in memory)
private var queueLength: Long = 0
@@ -110,7 +114,8 @@ class PersistentQueue(val name: String, persistencePath: String, @volatile var c
("age", currentAge.inMilliseconds.toString),
("discarded", totalDiscarded().toString),
("waiters", waiterCount.toString),
- ("open_transactions", openTransactionCount.toString)
+ ("open_transactions", openTransactionCount.toString),
+ ("total_flushes", totalFlushes().toString)
)
}
@@ -327,6 +332,7 @@ class PersistentQueue(val name: String, persistencePath: String, @volatile var c
def flush() {
while (remove(false).isDefined) { }
+ totalFlushes.incr()
}
/**
@@ -371,6 +377,7 @@ class PersistentQueue(val name: String, persistencePath: String, @volatile var c
Stats.removeCounter(statNamed("total_items"))
Stats.removeCounter(statNamed("expired_items"))
Stats.removeCounter(statNamed("discarded"))
+ Stats.removeCounter(statNamed("total_flushes"))
Stats.clearGauge(statNamed("items"))
Stats.clearGauge(statNamed("bytes"))
Stats.clearGauge(statNamed("journal_size"))
View
2  src/main/scala/net/lag/kestrel/QueueCollection.scala
@@ -54,6 +54,7 @@ class QueueCollection(queueFolder: String, timer: Timer, journalSyncScheduler: S
}
val config = queueConfigMap.getOrElse(name, defaultQueueConfig)
log.info("Setting up queue %s: %s", realName, config)
+ Stats.incr("queue_creates")
new PersistentQueue(realName, path, config, timer, journalSyncScheduler, Some(this.apply))
}
@@ -178,6 +179,7 @@ class QueueCollection(queueFolder: String, timer: Timer, journalSyncScheduler: S
q.destroyJournal()
q.removeStats()
queues.remove(name)
+ Stats.incr("queue_deletes")
}
if (name contains '+') {
val master = name.split('+')(0)
View
3  src/test/scala/net/lag/kestrel/PersistentQueueSpec.scala
@@ -95,6 +95,7 @@ class PersistentQueueSpec extends Specification
q.totalItems() mustEqual 0
q.bytes mustEqual 0
q.journalSize mustEqual 0
+ q.totalFlushes() mustEqual 0
q.add("alpha".getBytes)
q.add("beta".getBytes)
@@ -103,6 +104,8 @@ class PersistentQueueSpec extends Specification
q.flush()
q.length mustEqual 0
+
+ q.totalFlushes() mustEqual 1
// journal should contain exactly: one unfinished transaction, 2 items.
q.close
View
7 src/test/scala/net/lag/kestrel/QueueCollectionSpec.scala
@@ -47,6 +47,8 @@ class QueueCollectionSpec extends Specification with TempFolder with TestLogging
Stats.clearAll()
qc = new QueueCollection(folderName, timer, scheduler, config, Nil)
qc.queueNames mustEqual Nil
+ Stats.getCounter("queue_creates")() mustEqual 0
+ Stats.getCounter("queue_deletes")() mustEqual 0
qc.add("work1", "stuff".getBytes)
qc.add("work2", "other stuff".getBytes)
@@ -55,6 +57,8 @@ class QueueCollectionSpec extends Specification with TempFolder with TestLogging
qc.currentBytes mustEqual 16
qc.currentItems mustEqual 2
Stats.getCounter("total_items")() mustEqual 2
+ Stats.getCounter("queue_creates")() mustEqual 2
+ Stats.getCounter("queue_deletes")() mustEqual 0
qc.remove("work1")() must beSomeQItem("stuff")
qc.remove("work1")() mustEqual None
@@ -164,8 +168,11 @@ class QueueCollectionSpec extends Specification with TempFolder with TestLogging
new File(folderName + "/apples").createNewFile()
new File(folderName + "/oranges").createNewFile()
qc = new QueueCollection(folderName, timer, scheduler, config, Nil)
+ Stats.getCounter("queue_deletes")() mustEqual 0
qc.loadQueues()
qc.delete("oranges")
+
+ Stats.getCounter("queue_deletes")() mustEqual 1
new File(folderName).list().toList.sorted mustEqual List("apples")
qc.queueNames.sorted mustEqual List("apples")
Something went wrong with that request. Please try again.