Permalink
Browse files

Using Histogram conditionally (only with StoreStats). Updated histogr…

…am bucket size to 10000
  • Loading branch information...
1 parent 667da20 commit 72667991cd5b654703f613109d40f3fa1c416b35 Chinmay Soman committed Mar 15, 2012
Showing with 103 additions and 44 deletions.
  1. +76 −28 src/java/voldemort/store/stats/RequestCounter.java
  2. +27 −16 src/java/voldemort/store/stats/StoreStats.java
@@ -19,25 +19,43 @@
private final Histogram histogram;
private volatile int q95LatencyMs;
private volatile int q99LatencyMs;
+ private boolean withHistogram;
/**
* @param durationMS specifies for how long you want to maintain this
* counter (in milliseconds).
*/
public RequestCounter(int durationMS) {
- this(durationMS, SystemTime.INSTANCE);
+ this(durationMS, SystemTime.INSTANCE, false);
+ }
+
+ /**
+ * @param durationMS specifies for how long you want to maintain this
+ * counter (in milliseconds). withHistogram indicates that this
+ * counter should also use a histogram.
+ */
+ public RequestCounter(int durationMS, boolean withHistogram) {
+ this(durationMS, SystemTime.INSTANCE, withHistogram);
}
/**
* For testing request expiration via an injected time provider
*/
RequestCounter(int durationMS, Time time) {
+ this(durationMS, time, false);
+ }
+
+ RequestCounter(int durationMS, Time time, boolean withHistogram) {
this.time = time;
this.values = new AtomicReference<Accumulator>(new Accumulator());
this.durationMS = durationMS;
- this.histogram = new Histogram(65535, 1);
this.q95LatencyMs = 0;
this.q99LatencyMs = 0;
+ this.withHistogram = withHistogram;
+ if(this.withHistogram)
+ this.histogram = new Histogram(10000, 1);
+ else
+ this.histogram = null;
}
public long getCount() {
@@ -50,8 +68,7 @@ public long getTotalCount() {
public float getThroughput() {
Accumulator oldv = getValidAccumulator();
- double elapsed = (time.getMilliseconds() - oldv.startTimeMS)
- / (double) Time.MS_PER_SECOND;
+ double elapsed = (time.getMilliseconds() - oldv.startTimeMS) / (double) Time.MS_PER_SECOND;
if(elapsed > 0f) {
return (float) (oldv.count / elapsed);
} else {
@@ -61,8 +78,7 @@ public float getThroughput() {
public float getThroughputInBytes() {
Accumulator oldv = getValidAccumulator();
- double elapsed = (time.getMilliseconds() - oldv.startTimeMS)
- / (double) Time.MS_PER_SECOND;
+ double elapsed = (time.getMilliseconds() - oldv.startTimeMS) / (double) Time.MS_PER_SECOND;
if(elapsed > 0f) {
return (float) (oldv.totalBytes / elapsed);
} else {
@@ -91,10 +107,12 @@ public long getMaxLatencyInMs() {
}
private void maybeResetHistogram() {
+ if(!this.withHistogram)
+ return;
Accumulator accum = values.get();
long now = time.getMilliseconds();
if(now - accum.startTimeMS > durationMS) {
- // Reset the histogram
+ // Reset the histogram
q95LatencyMs = histogram.getQuantile(0.95);
q99LatencyMs = histogram.getQuantile(0.99);
histogram.reset();
@@ -140,17 +158,24 @@ public void addRequest(long timeNS) {
}
/**
- * @see #addRequest(long)
- * Detailed request to track additionald data about PUT, GET and GET_ALL
- *
- * @param numEmptyResponses For GET and GET_ALL, how many keys were no values found
+ * @see #addRequest(long) Detailed request to track additionald data about
+ * PUT, GET and GET_ALL
+ *
+ * @param numEmptyResponses For GET and GET_ALL, how many keys were no
+ * values found
* @param bytes Total number of bytes across all versions of values' bytes
- * @param getAllAggregatedCount Total number of keys returned for getAll calls
+ * @param getAllAggregatedCount Total number of keys returned for getAll
+ * calls
*/
- public void addRequest(long timeNS, long numEmptyResponses, long bytes, long getAllAggregatedCount) {
+ public void addRequest(long timeNS,
+ long numEmptyResponses,
+ long bytes,
+ long getAllAggregatedCount) {
int timeMs = (int) timeNS / (int) Time.NS_PER_MS;
- histogram.insert(timeMs);
- maybeResetHistogram();
+ if(this.withHistogram) {
+ histogram.insert(timeMs);
+ maybeResetHistogram();
+ }
for(int i = 0; i < 3; i++) {
Accumulator oldv = getValidAccumulator();
Accumulator newv = new Accumulator(oldv.startTimeMS,
@@ -168,28 +193,32 @@ public void addRequest(long timeNS, long numEmptyResponses, long bytes, long get
}
/**
- * Return the number of requests that have returned returned no value for the requested key. Tracked only for GET.
+ * Return the number of requests that have returned returned no value for
+ * the requested key. Tracked only for GET.
*/
public long getNumEmptyResponses() {
return getValidAccumulator().numEmptyResponses;
}
/**
- * Return the size of the largest response or request in bytes returned. Tracked only for GET, GET_ALL and PUT.
+ * Return the size of the largest response or request in bytes returned.
+ * Tracked only for GET, GET_ALL and PUT.
*/
public long getMaxSizeInBytes() {
return getValidAccumulator().maxBytes;
}
/**
- * Return the average size of all the versioned values returned. Tracked only for GET, GET_ALL and PUT.
+ * Return the average size of all the versioned values returned. Tracked
+ * only for GET, GET_ALL and PUT.
*/
public double getAverageSizeInBytes() {
return getValidAccumulator().getAverageBytes();
}
/**
- * Return the aggregated number of keys returned across all getAll calls, taking into account multiple values returned per call.
+ * Return the aggregated number of keys returned across all getAll calls,
+ * taking into account multiple values returned per call.
*/
public long getGetAllAggregatedCount() {
return getValidAccumulator().getAllAggregatedCount;
@@ -198,32 +227,51 @@ public long getGetAllAggregatedCount() {
public int getQ95LatencyMs() {
return q95LatencyMs;
}
-
+
public int getQ99LatencyMs() {
return q99LatencyMs;
- }
-
+ }
+
private class Accumulator {
final long startTimeMS;
final long count;
final long totalTimeNS;
final long total;
- final long numEmptyResponses; // GET and GET_ALL: number of empty responses that have been returned
- final long getAllAggregatedCount; // GET_ALL: a single call to GET_ALL can return multiple k-v pairs. Track total returned.
+ final long numEmptyResponses; // GET and GET_ALL: number of empty
+ // responses that have been returned
+ final long getAllAggregatedCount; // GET_ALL: a single call to GET_ALL
+ // can return multiple k-v pairs.
+ // Track total returned.
final long maxLatencyNS;
- final long maxBytes; // Maximum single value
- final long totalBytes; // Sum of all the values
+ final long maxBytes; // Maximum single value
+ final long totalBytes; // Sum of all the values
public Accumulator() {
this(RequestCounter.this.time.getMilliseconds(), 0, 0, 0, 0, 0, 0, 0, 0);
}
public Accumulator newWithTotal() {
- return new Accumulator(RequestCounter.this.time.getMilliseconds(), 0, 0, total, 0, 0, 0, 0, 0);
+ return new Accumulator(RequestCounter.this.time.getMilliseconds(),
+ 0,
+ 0,
+ total,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0);
}
- public Accumulator(long startTimeMS, long count, long totalTimeNS, long total, long numEmptyResponses, long maxLatencyNS, long totalBytes, long maxBytes, long getAllAggregatedCount) {
+ public Accumulator(long startTimeMS,
+ long count,
+ long totalTimeNS,
+ long total,
+ long numEmptyResponses,
+ long maxLatencyNS,
+ long totalBytes,
+ long maxBytes,
+ long getAllAggregatedCount) {
this.startTimeMS = startTimeMS;
this.count = count;
this.totalTimeNS = totalTimeNS;
@@ -26,51 +26,62 @@ public StoreStats(StoreStats parent) {
counters = new EnumMap<Tracked, RequestCounter>(Tracked.class);
for(Tracked tracked: Tracked.values()) {
- counters.put(tracked, new RequestCounter(300000));
+ counters.put(tracked, new RequestCounter(300000, true));
}
this.parent = parent;
}
/**
- * Record the duration of specified op. For PUT, GET and GET_ALL use specific methods for those ops.
+ * Record the duration of specified op. For PUT, GET and GET_ALL use
+ * specific methods for those ops.
*/
public void recordTime(Tracked op, long timeNS) {
recordTime(op, timeNS, 0, 0, 0);
}
/**
- * Record the duration of a put operation, along with the size of the values returned.
+ * Record the duration of a put operation, along with the size of the values
+ * returned.
*/
public void recordPutTimeAndSize(long timeNS, long size) {
- recordTime(Tracked.PUT, timeNS, 0, size, 0);
+ recordTime(Tracked.PUT, timeNS, 0, size, 0);
}
/**
- * Record the duration of a get operation, along with whether or not an empty response (ie no values matched)
- * and the size of the values returned.
+ * Record the duration of a get operation, along with whether or not an
+ * empty response (ie no values matched) and the size of the values
+ * returned.
*/
public void recordGetTime(long timeNS, boolean emptyResponse, long totalBytes) {
recordTime(Tracked.GET, timeNS, emptyResponse ? 1 : 0, totalBytes, 0);
}
/**
- * Record the duration of a get_all operation, along with how many values were requested, how may were actually
- * returned and the size of the values returned.
+ * Record the duration of a get_all operation, along with how many values
+ * were requested, how may were actually returned and the size of the values
+ * returned.
*/
public void recordGetAllTime(long timeNS, int requested, int returned, long totalBytes) {
recordTime(Tracked.GET_ALL, timeNS, requested - returned, totalBytes, requested);
}
/**
* Method to service public recording APIs
- *
- * @param op Operation being tracked
- * @param timeNS Duration of operation
- * @param numEmptyResponses GET and GET_ALL: number of empty responses being sent back, ie requested keys for which there were no values
- * @param size Total size of response payload, ie sum of lengths of bytes in all versions of values
- * @param getAllAggregateRequests Total of key-values requested in aggregatee from get_all operations
+ *
+ * @param op Operation being tracked
+ * @param timeNS Duration of operation
+ * @param numEmptyResponses GET and GET_ALL: number of empty responses being
+ * sent back, ie requested keys for which there were no values
+ * @param size Total size of response payload, ie sum of lengths of bytes in
+ * all versions of values
+ * @param getAllAggregateRequests Total of key-values requested in
+ * aggregatee from get_all operations
*/
- private void recordTime(Tracked op, long timeNS, long numEmptyResponses, long size, long getAllAggregateRequests) {
+ private void recordTime(Tracked op,
+ long timeNS,
+ long numEmptyResponses,
+ long size,
+ long getAllAggregateRequests) {
counters.get(op).addRequest(timeNS, numEmptyResponses, size, getAllAggregateRequests);
if(parent != null)
parent.recordTime(op, timeNS, numEmptyResponses, size, getAllAggregateRequests);
@@ -103,7 +114,7 @@ public long getMaxLatencyInMs(Tracked op) {
public long getQ95LatencyInMs(Tracked op) {
return counters.get(op).getQ95LatencyMs();
}
-
+
public long getQ99LatencyInMs(Tracked op) {
return counters.get(op).getQ99LatencyMs();
}

0 comments on commit 7266799

Please sign in to comment.