Skip to content
Permalink
Browse files
Add units to performance test results (#156)
* Add units to PT output
* Added constants and corrections
* Updated results descriptions
  • Loading branch information
DomGarguilo committed Sep 24, 2021
1 parent 6e792d9 commit 86536bb7414427c9f4146188788f1dfabd069892
Show file tree
Hide file tree
Showing 14 changed files with 159 additions and 125 deletions.
@@ -55,35 +55,37 @@ public Builder description(String desc) {
return this;
}

public Builder result(String id, LongSummaryStatistics stats, String description) {
public Builder result(String id, LongSummaryStatistics stats, String units,
String description) {
results.add(new Result(id, new Stats(stats.getMin(), stats.getMax(), stats.getSum(),
stats.getAverage(), stats.getCount()), description, Purpose.COMPARISON));
stats.getAverage(), stats.getCount()), units, description, Purpose.COMPARISON));
return this;
}

public Builder result(String id, Number data, String description) {
results.add(new Result(id, data, description, Purpose.COMPARISON));
public Builder result(String id, Number data, String units, String description) {
results.add(new Result(id, data, units, description, Purpose.COMPARISON));
return this;
}

public Builder result(String id, long amount, long time, String description) {
results.add(new Result(id, amount / (time / 1000.0), description, Purpose.COMPARISON));
public Builder result(String id, long amount, long time, String units, String description) {
results.add(new Result(id, amount / (time / 1000.0), units, description, Purpose.COMPARISON));
return this;
}

public Builder info(String id, LongSummaryStatistics stats, String description) {
public Builder info(String id, LongSummaryStatistics stats, String units, String description) {
results.add(new Result(id, new Stats(stats.getMin(), stats.getMax(), stats.getSum(),
stats.getAverage(), stats.getCount()), description, Purpose.INFORMATIONAL));
stats.getAverage(), stats.getCount()), units, description, Purpose.INFORMATIONAL));
return this;
}

public Builder info(String id, long amount, long time, String description) {
results.add(new Result(id, amount / (time / 1000.0), description, Purpose.INFORMATIONAL));
public Builder info(String id, long amount, long time, String units, String description) {
results
.add(new Result(id, amount / (time / 1000.0), units, description, Purpose.INFORMATIONAL));
return this;
}

public Builder info(String id, Number data, String description) {
results.add(new Result(id, data, description, Purpose.INFORMATIONAL));
public Builder info(String id, Number data, String units, String description) {
results.add(new Result(id, data, units, description, Purpose.INFORMATIONAL));
return this;
}

@@ -22,6 +22,7 @@ public class Result {
public final String id;
public final Number data;
public final Stats stats;
public final String units;
public final String description;
public final Purpose purpose;

@@ -36,18 +37,20 @@ public enum Purpose {
COMPARISON
}

public Result(String id, Number data, String description, Purpose purpose) {
public Result(String id, Number data, String units, String description, Purpose purpose) {
this.id = id;
this.data = data;
this.stats = null;
this.units = units;
this.description = description;
this.purpose = purpose;
}

public Result(String id, Stats stats, String description, Purpose purpose) {
public Result(String id, Stats stats, String units, String description, Purpose purpose) {
this.id = id;
this.stats = stats;
this.data = null;
this.stats = stats;
this.units = units;
this.description = description;
this.purpose = purpose;
}
@@ -55,6 +55,8 @@

public class ConditionalMutationsPT implements PerformanceTest {

private static final String conditionsPerSec = "conditions/sec";

@Override
public SystemConfiguration getSystemConfig() {
Map<String,String> siteCfg = new HashMap<>();
@@ -102,8 +104,8 @@ private static void runConditionalMutationsTest(Environment env, String tableNam
}

reportBuilder.result("avgRate: 1-20",
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
"ConditionalMutationsTest: average rate (conditions/sec) to run sequence 1-20");
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)), conditionsPerSec,
"ConditionalMutationsTest: average rate to run sequence 1-20");

env.getClient().tableOperations().flush(tableName, null, null, true);

@@ -113,8 +115,8 @@ private static void runConditionalMutationsTest(Environment env, String tableNam
}

reportBuilder.result("avgRate: 21-40",
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
"ConditionalMutationsTest: average rate (conditions/sec) to run sequence 21-40");
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)), conditionsPerSec,
"ConditionalMutationsTest: average rate to run sequence 21-40");
}

public static double conditionalMutationsTime(ConditionalWriter cw, long seq) throws Exception {
@@ -178,8 +180,8 @@ private static void runRandomizeConditionalMutationsTest(Environment env, String
}

reportBuilder.result("avgRate: 1-20",
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
"RandomizeConditionalMutationsTest: average rate (conditions/sec) to run sequence 1-20");
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)), conditionsPerSec,
"RandomizeConditionalMutationsTest: average rate to run sequence 1-20");

env.getClient().tableOperations().flush(tableName, null, null, true);

@@ -189,8 +191,8 @@ private static void runRandomizeConditionalMutationsTest(Environment env, String
}

reportBuilder.result("avgRate: 21-40",
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
"RandomizeConditionalMutationsTest: average rate (conditions/sec) to run sequence 21-40");
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)), conditionsPerSec,
"RandomizeConditionalMutationsTest: average rate to run sequence 21-40");
}

private static double randomizeConditionalMutationsTime(ConditionalWriter cw, long seq)
@@ -266,8 +268,8 @@ private static void runRandomizeBatchScanAndWriteTest(Environment env, String ta
}

reportBuilder.result("avgRate: 1-20",
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
"RandomizeBatchScanAndWriteTest: average rate (conditions/sec) to write and scan sequence 1-20");
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)), conditionsPerSec,
"RandomizeBatchScanAndWriteTest: average rate to write and scan sequence 1-20");

env.getClient().tableOperations().flush(tableName, null, null, true);

@@ -277,8 +279,8 @@ private static void runRandomizeBatchScanAndWriteTest(Environment env, String ta
}

reportBuilder.result("avgRate: 21-40",
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
"RandomizeBatchScanAndWriteTest: average rate (conditions/sec) to write and scan sequence 21-40 post flush");
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)), conditionsPerSec,
"RandomizeBatchScanAndWriteTest: average rate to write and scan sequence 21-40 post flush");
}

private static double randomizeBatchWriteAndScanTime(BatchWriter bw, BatchScanner bs, long seq)
@@ -349,8 +351,8 @@ private static void runSetBlockSizeTest(Environment env, String tableName,
}

reportBuilder.result("avgRate1",
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / numTest)),
"SetBlockSizeTest: average rate in conditions/sec");
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / numTest)), conditionsPerSec,
"SetBlockSizeTest: average rate");

env.getClient().tableOperations().flush(tableName, null, null, true);

@@ -360,8 +362,8 @@ private static void runSetBlockSizeTest(Environment env, String tableName,
}

reportBuilder.result("avgRate2",
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / numTest)),
"SetBlockSizeTest: average rate in conditions/sec post flush");
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / numTest)), conditionsPerSec,
"SetBlockSizeTest: average rate post flush");

env.getClient().tableOperations().compact(tableName, null, null, true, true);

@@ -371,8 +373,8 @@ private static void runSetBlockSizeTest(Environment env, String tableName,
}

reportBuilder.result("avgRate3",
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)),
"SetBlockSizeTest: average rate in conditions/sec post compaction");
Double.parseDouble(new DecimalFormat("#0.00").format(rateSum / 20)), conditionsPerSec,
"SetBlockSizeTest: average rate post compaction");
reportBuilder.parameter("numRows", numRows, "SetBlockSizeTest: The number of rows");
reportBuilder.parameter("numCols", numCols, "SetBlockSizeTest: The number of columns");
reportBuilder.parameter("numTest", numTest,
@@ -19,6 +19,7 @@

import java.util.Arrays;
import java.util.Collections;
import java.util.concurrent.TimeUnit;

import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchWriter;
@@ -56,7 +57,7 @@ public Report runTest(Environment env) throws Exception {
createTable(client, tableName, durability);
long median = writeSome(reportBuilder, client, tableName, N, durability);
tableOps.delete(tableName);
reportBuilder.result(durability + " Median", median,
reportBuilder.result(durability + " Median", median, TimeUnit.MILLISECONDS.toString(),
"Median time result for " + durability);
}
}
@@ -82,7 +83,7 @@ private long writeSome(Report.Builder reportBuilder, AccumuloClient c, String ta
}
attempts[attempt] = System.currentTimeMillis() - now;
reportBuilder.info(durabilityLevel + " attempt " + attempt, System.currentTimeMillis() - now,
"Times for each attempt in ms");
TimeUnit.MILLISECONDS.toString(), "Times for each attempt");
}
Arrays.sort(attempts);
// Return the median duration
@@ -222,11 +222,11 @@ private void runTest(Report.Builder report, Environment env, int numThreads, boo

env.getClient().tableOperations().delete(tableName);
if (warmup) {
report.info("warmup_rate_" + numThreads, NUM_MUTATIONS, t2 - t1, "The warmup rate at which "
+ numThreads + " threads wrote data. The rate is mutations per second.");
report.info("warmup_rate_" + numThreads, NUM_MUTATIONS, t2 - t1, "mutations/sec",
"The warmup rate at which " + numThreads + " threads wrote data.");
} else {
report.result("rate_" + numThreads, NUM_MUTATIONS, t2 - t1, "The rate at which " + numThreads
+ " threads wrote data. The rate is mutations per second.");
report.result("rate_" + numThreads, NUM_MUTATIONS, t2 - t1, "mutations/sec",
"The rate at which " + numThreads + " threads wrote data.");
}
}

@@ -27,6 +27,7 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
@@ -86,7 +87,8 @@ public Report runTest(final Environment env) throws Exception {
final AccumuloClient client = env.getClient();
initTable(client);
long herdTime = getHerdingDuration(client);
reportBuilder.result("herd_time", herdTime, "The time (in ms) it took herding to complete.");
reportBuilder.result("herd_time", herdTime, TimeUnit.MILLISECONDS.toString(),
"The time it took herding to complete.");

return reportBuilder.build();
}
@@ -111,10 +113,10 @@ private Mutation createMutation(final int rowNum) {
byte[] row = toZeroPaddedString(rowNum, 8);
Mutation mutation = new Mutation(row);
for (int col = 0; col < NUM_COLS; col++) {
byte[] qualifer = toZeroPaddedString(col, 4);
byte[] qualifier = toZeroPaddedString(col, 4);
byte[] value = new byte[32];
random.nextBytes(value);
mutation.put(COL_FAM, qualifer, value);
mutation.put(COL_FAM, qualifier, value);
}
return mutation;
}
@@ -62,8 +62,8 @@ public Report runTest(final Environment env) throws Exception {
long totalTime = System.currentTimeMillis() - start;
double splitsPerSecond = NUM_SPLITS / (totalTime / ONE_SECOND);

reportBuilder.result("splits_per_second", splitsPerSecond,
"The average number of splits created per second.");
reportBuilder.result("splits_per_second", splitsPerSecond, "splits/sec",
"The average rate of split creation.");

return reportBuilder.build();
}
@@ -29,6 +29,7 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;

import org.apache.accumulo.core.client.AccumuloClient;
@@ -57,6 +58,8 @@ public class RandomCachedLookupsPT implements PerformanceTest {
private static final int NUM_LOOKUPS_PER_THREAD = 25000;
private static final int NUM_ROWS = 100000;

private static final String ms = TimeUnit.MILLISECONDS.toString();

@Override
public SystemConfiguration getSystemConfig() {
Map<String,String> siteCfg = new HashMap<>();
@@ -89,40 +92,42 @@ public Report runTest(Environment env) throws Exception {
long d64 = doLookups(env.getClient(), 64, NUM_LOOKUPS_PER_THREAD);
long d128 = doLookups(env.getClient(), 128, NUM_LOOKUPS_PER_THREAD);

final String lookupPerSec = "lookups/sec";

reportBuilder.id("smalls");
reportBuilder.description(
"Runs multiple threads each doing lots of small random scans. For this test data and index cache are enabled.");
reportBuilder.info("warmup", 32 * NUM_LOOKUPS_PER_THREAD, warmup,
"Random lookup per sec for 32 threads");
reportBuilder.info("lookups_1", NUM_LOOKUPS_PER_THREAD, d1,
"Random lookup per sec rate for 1 thread");
reportBuilder.info("lookups_4", 4 * NUM_LOOKUPS_PER_THREAD, d4,
"Random lookup per sec rate for 4 threads");
reportBuilder.info("lookups_8", 8 * NUM_LOOKUPS_PER_THREAD, d8,
"Random lookup per sec rate for 8 threads");
reportBuilder.info("lookups_16", 16 * NUM_LOOKUPS_PER_THREAD, d16,
"Random lookup per sec rate for 16 threads");
reportBuilder.info("lookups_32", 32 * NUM_LOOKUPS_PER_THREAD, d32,
"Random lookup per sec rate for 32 threads");
reportBuilder.info("lookups_64", 64 * NUM_LOOKUPS_PER_THREAD, d64,
"Random lookup per sec rate for 64 threads");
reportBuilder.info("lookups_128", 128 * NUM_LOOKUPS_PER_THREAD, d128,
"Random lookup per sec rate for 128 threads");

reportBuilder.result("avg_1", d1 / (double) NUM_LOOKUPS_PER_THREAD,
"Average milliseconds per lookup for 1 thread");
reportBuilder.result("avg_4", d4 / (double) NUM_LOOKUPS_PER_THREAD,
"Average milliseconds per lookup for 4 threads");
reportBuilder.result("avg_8", d8 / (double) NUM_LOOKUPS_PER_THREAD,
"Average milliseconds per lookup for 8 threads");
reportBuilder.result("avg_16", d16 / (double) NUM_LOOKUPS_PER_THREAD,
"Average milliseconds per lookup for 16 threads");
reportBuilder.result("avg_32", d32 / (double) NUM_LOOKUPS_PER_THREAD,
"Average milliseconds per lookup for 32 threads");
reportBuilder.result("avg_64", d64 / (double) NUM_LOOKUPS_PER_THREAD,
"Average milliseconds per lookup for 64 threads");
reportBuilder.result("avg_128", d128 / (double) NUM_LOOKUPS_PER_THREAD,
"Average milliseconds per lookup for 128 threads");
reportBuilder.info("warmup", 32 * NUM_LOOKUPS_PER_THREAD, warmup, lookupPerSec,
"Random lookup rate for 32 threads");
reportBuilder.info("lookups_1", NUM_LOOKUPS_PER_THREAD, d1, lookupPerSec,
"Random lookup rate for 1 thread");
reportBuilder.info("lookups_4", 4 * NUM_LOOKUPS_PER_THREAD, d4, lookupPerSec,
"Random lookup rate for 4 threads");
reportBuilder.info("lookups_8", 8 * NUM_LOOKUPS_PER_THREAD, d8, lookupPerSec,
"Random lookup rate for 8 threads");
reportBuilder.info("lookups_16", 16 * NUM_LOOKUPS_PER_THREAD, d16, lookupPerSec,
"Random lookup rate for 16 threads");
reportBuilder.info("lookups_32", 32 * NUM_LOOKUPS_PER_THREAD, d32, lookupPerSec,
"Random lookup rate for 32 threads");
reportBuilder.info("lookups_64", 64 * NUM_LOOKUPS_PER_THREAD, d64, lookupPerSec,
"Random lookup rate for 64 threads");
reportBuilder.info("lookups_128", 128 * NUM_LOOKUPS_PER_THREAD, d128, lookupPerSec,
"Random lookup rate for 128 threads");

reportBuilder.result("avg_1", d1 / (double) NUM_LOOKUPS_PER_THREAD, ms,
"Average duration per lookup for 1 thread");
reportBuilder.result("avg_4", d4 / (double) NUM_LOOKUPS_PER_THREAD, ms,
"Average duration per lookup for 4 threads");
reportBuilder.result("avg_8", d8 / (double) NUM_LOOKUPS_PER_THREAD, ms,
"Average duration per lookup for 8 threads");
reportBuilder.result("avg_16", d16 / (double) NUM_LOOKUPS_PER_THREAD, ms,
"Average duration per lookup for 16 threads");
reportBuilder.result("avg_32", d32 / (double) NUM_LOOKUPS_PER_THREAD, ms,
"Average duration per lookup for 32 threads");
reportBuilder.result("avg_64", d64 / (double) NUM_LOOKUPS_PER_THREAD, ms,
"Average duration per lookup for 64 threads");
reportBuilder.result("avg_128", d128 / (double) NUM_LOOKUPS_PER_THREAD, ms,
"Average duration per lookup for 128 threads");

return reportBuilder.build();
}
@@ -196,12 +201,14 @@ public static void writeData(Report.Builder reportBuilder, AccumuloClient client

long t6 = System.currentTimeMillis();

reportBuilder.info("create", t2 - t1, "Time to create table in ms");
reportBuilder.info("split", t3 - t2, "Time to split table in ms");
reportBuilder.info("write", 4 * numRows, t4 - t3, "Rate to write data in entries/sec");
reportBuilder.info("compact", 4 * numRows, t5 - t4, "Rate to compact table in entries/sec");
reportBuilder.info("fullScan", 4 * numRows, t6 - t5,
"Rate to do full table scan in entries/sec");
reportBuilder.info("create", t2 - t1, ms, "Time to create table");
reportBuilder.info("split", t3 - t2, ms, "Time to split table");
reportBuilder.info("write", 4 * numRows, t4 - t3, "entries/sec",
"Rate at which data are written");
reportBuilder.info("compact", 4 * numRows, t5 - t4, "entries/sec",
"Rate at which tables are compacted");
reportBuilder.info("fullScan", 4 * numRows, t6 - t5, "entries/sec",
"Rate at which full table scans take place");
}

private static long doLookups(AccumuloClient client, int numThreads, int numScansPerThread)

0 comments on commit 86536bb

Please sign in to comment.