From 6e201764a5de5b477442309586e1784fac1c7906 Mon Sep 17 00:00:00 2001 From: Tim Armstrong Date: Wed, 6 Mar 2013 14:24:02 -0600 Subject: [PATCH] Add linter to arcanist config. Fix whitespace errors Summary: Arcanist text linter to detect basic formatting problems like tabs and trailing whitespace. Clean up whitespace errors to avoid noise in future diffs from whitespace fixes Test Plan: Still compiles ok Reviewers: emayanke, dhruba Reviewed By: emayanke Differential Revision: https://reviews.facebook.net/D8301 --- .arcconfig | 4 +- src/java/com/facebook/LinkBench/Config.java | 24 +- .../com/facebook/LinkBench/ConfigUtil.java | 66 ++--- .../com/facebook/LinkBench/GraphStore.java | 2 +- .../LinkBench/InvertibleShuffler.java | 20 +- src/java/com/facebook/LinkBench/Link.java | 10 +- .../facebook/LinkBench/LinkBenchDriver.java | 141 +++++----- .../facebook/LinkBench/LinkBenchDriverMR.java | 16 +- .../com/facebook/LinkBench/LinkBenchLoad.java | 128 ++++----- .../com/facebook/LinkBench/LinkBenchOp.java | 6 +- .../facebook/LinkBench/LinkBenchRequest.java | 230 ++++++++-------- .../com/facebook/LinkBench/LinkCount.java | 2 +- .../com/facebook/LinkBench/LinkStore.java | 46 ++-- .../facebook/LinkBench/LinkStoreMysql.java | 138 +++++----- .../facebook/LinkBench/LinkStoreRocksDb.java | 76 ++--- .../facebook/LinkBench/MemoryLinkStore.java | 68 ++--- src/java/com/facebook/LinkBench/Node.java | 4 +- .../com/facebook/LinkBench/NodeLoader.java | 62 ++--- .../com/facebook/LinkBench/NodeStore.java | 26 +- .../facebook/LinkBench/RealDistribution.java | 44 +-- src/java/com/facebook/LinkBench/Shuffler.java | 14 +- src/java/com/facebook/LinkBench/Timer.java | 4 +- .../distributions/AccessDistributions.java | 32 +-- .../distributions/ApproxHarmonic.java | 6 +- .../distributions/GeometricDistribution.java | 20 +- .../LinkBench/distributions/Harmonic.java | 2 +- .../LinkBench/distributions/ID2Chooser.java | 58 ++-- .../distributions/LinkDistributions.java | 10 +- .../distributions/LogNormalDistribution.java | 16 +- .../PiecewiseLinearDistribution.java | 48 ++-- .../ProbabilityDistribution.java | 14 +- .../distributions/UniformDistribution.java | 28 +- .../distributions/ZipfDistribution.java | 54 ++-- .../LinkBench/generators/DataGenerator.java | 6 +- .../generators/MotifDataGenerator.java | 53 ++-- .../generators/UniformDataGenerator.java | 12 +- .../LinkBench/stats/LatencyStats.java | 80 +++--- .../facebook/LinkBench/stats/RunningMean.java | 12 +- .../LinkBench/stats/SampledStats.java | 38 +-- .../LinkBench/util/ClassLoadUtil.java | 8 +- src/java/com/facebook/rocks/swift/Code.java | 2 +- .../facebook/rocks/swift/CompressionType.java | 2 +- .../com/facebook/rocks/swift/IOError.java | 2 +- .../com/facebook/rocks/swift/ReadOptions.java | 2 +- .../facebook/rocks/swift/ResultSnapshot.java | 2 +- .../rocks/swift/RocksIterateResponse.java | 2 +- .../com/facebook/rocks/swift/Snapshot.java | 2 +- .../facebook/rocks/swift/WriteOptions.java | 2 +- .../LinkBench/DistributionTestBase.java | 78 +++--- .../facebook/LinkBench/DummyLinkStore.java | 20 +- .../LinkBench/DummyLinkStoreTest.java | 3 +- .../facebook/LinkBench/GeneratedDataDump.java | 16 +- .../com/facebook/LinkBench/GeomDistTest.java | 2 +- .../LinkBench/GraphStoreTestBase.java | 90 +++--- .../com/facebook/LinkBench/HarmonicTest.java | 8 +- .../facebook/LinkBench/ID2ChooserTest.java | 40 +-- .../LinkBench/InvertibleShufflerTest.java | 26 +- .../facebook/LinkBench/LinkStoreTestBase.java | 260 +++++++++--------- .../com/facebook/LinkBench/LogNormalTest.java | 6 +- .../LinkBench/MemoryLinkStoreTest.java | 12 +- .../LinkBench/MySqlGraphStoreTest.java | 2 +- .../LinkBench/MySqlLinkStoreTest.java | 12 +- .../LinkBench/MySqlNodeStoreTest.java | 2 +- .../facebook/LinkBench/MySqlTestConfig.java | 22 +- .../facebook/LinkBench/NodeStoreTestBase.java | 54 ++-- .../facebook/LinkBench/PiecewiseDistTest.java | 12 +- .../LinkBench/TestAccessDistribution.java | 26 +- .../com/facebook/LinkBench/TestDataGen.java | 54 ++-- .../LinkBench/TestRealDistribution.java | 42 +-- .../com/facebook/LinkBench/TestStats.java | 2 +- .../com/facebook/LinkBench/TimerTest.java | 22 +- .../facebook/LinkBench/UniformDistTest.java | 4 +- .../com/facebook/LinkBench/ZipfDistTest.java | 14 +- 73 files changed, 1238 insertions(+), 1235 deletions(-) diff --git a/.arcconfig b/.arcconfig index 657b3f63..fa525624 100644 --- a/.arcconfig +++ b/.arcconfig @@ -1,6 +1,8 @@ { "project_id" : "linkbench", "conduit_uri" : "https://reviews.facebook.net/", - "copyright_holder" : "" + "copyright_holder" : "", + "lint.engine" : "ArcanistSingleLintEngine", + "lint.engine.single.linter" : "ArcanistTextLinter" } diff --git a/src/java/com/facebook/LinkBench/Config.java b/src/java/com/facebook/LinkBench/Config.java index d0bb05f1..09cfcb58 100644 --- a/src/java/com/facebook/LinkBench/Config.java +++ b/src/java/com/facebook/LinkBench/Config.java @@ -24,17 +24,17 @@ public class Config { public static final String DEBUGLEVEL = "debuglevel"; - + /* Control store implementations used */ public static final String LINKSTORE_CLASS = "linkstore"; public static final String NODESTORE_CLASS = "nodestore"; - + /* Schema and tables used */ public static final String DBID = "dbid"; public static final String LINK_TABLE = "linktable"; public static final String COUNT_TABLE = "counttable"; public static final String NODE_TABLE = "nodetable"; - + /* Control graph structure */ public static final String LOAD_RANDOM_SEED = "load_random_seed"; public static final String MIN_ID = "startid1"; @@ -46,7 +46,7 @@ public class Config { public static final String NLINKS_CONFIG = "nlinks_config"; public static final String NLINKS_DEFAULT = "nlinks_default"; public static final String LINK_TYPE_COUNT ="link_type_count"; - + /* Data generation */ public static final String LINK_DATASIZE = "link_datasize"; public static final String NODE_DATASIZE = "node_datasize"; @@ -69,11 +69,11 @@ public class Config { /* Loading performance tuning */ public static final String NUM_LOADERS = "loaders"; public static final String LOADER_CHUNK_SIZE = "loader_chunk_size"; - + /* Request workload */ public static final String NUM_REQUESTERS = "requesters"; public static final String REQUEST_RANDOM_SEED = "request_random_seed"; - + // Distribution of accesses to IDs public static final String READ_CONFIG_PREFIX = "read_"; public static final String WRITE_CONFIG_PREFIX = "write_"; @@ -93,11 +93,11 @@ public class Config { public static final String WRITE_UNCORR_FUNCTION = WRITE_UNCORR_CONFIG_PREFIX + ACCESS_FUNCTION_SUFFIX; public static final String BLEND_SUFFIX = "blend"; - public static final String READ_UNCORR_BLEND = READ_UNCORR_CONFIG_PREFIX + public static final String READ_UNCORR_BLEND = READ_UNCORR_CONFIG_PREFIX + BLEND_SUFFIX; public static final String WRITE_UNCORR_BLEND = WRITE_UNCORR_CONFIG_PREFIX + BLEND_SUFFIX; - + // Probability of different operations public static final String PR_ADD_LINK = "addlink"; public static final String PR_DELETE_LINK = "deletelink"; @@ -120,20 +120,20 @@ public class Config { public static final String LINK_MULTIGET_DIST_MIN = "link_multiget_dist_min"; public static final String LINK_MULTIGET_DIST_MAX = "link_multiget_dist_max"; public static final String LINK_MULTIGET_DIST_PREFIX = "link_multiget_dist_"; - + /* Probability distribution parameters */ public static final String PROB_MEAN = "mean"; - + /* Statistics collection and reporting */ public static final String MAX_STAT_SAMPLES = "maxsamples"; public static final String DISPLAY_FREQ = "displayfreq"; public static final String MAPRED_REPORT_PROGRESS = "reportprogress"; public static final String PROGRESS_FREQ = "progressfreq"; - + /* Reporting for progress indicators */ public static String REQ_PROG_INTERVAL = "req_progress_interval"; public static String LOAD_PROG_INTERVAL = "load_progress_interval"; - + /* MapReduce specific configuration */ public static final String TEMPDIR = "tempdir"; public static final String LOAD_DATA = "loaddata"; diff --git a/src/java/com/facebook/LinkBench/ConfigUtil.java b/src/java/com/facebook/LinkBench/ConfigUtil.java index 41b268d6..cba50c5a 100644 --- a/src/java/com/facebook/LinkBench/ConfigUtil.java +++ b/src/java/com/facebook/LinkBench/ConfigUtil.java @@ -29,9 +29,9 @@ public class ConfigUtil { public static final String linkbenchHomeEnvVar = "LINKBENCH_HOME"; public static final String LINKBENCH_LOGGER = "com.facebook.linkbench"; - + /** - * @return null if not set, or if not valid path + * @return null if not set, or if not valid path */ public static String findLinkBenchHome() { String linkBenchHome = System.getenv("LINKBENCH_HOME"); @@ -43,18 +43,18 @@ public static String findLinkBenchHome() { } return null; } - + public static Level getDebugLevel(Properties props) throws LinkBenchConfigError { if (props == null) { return Level.DEBUG; } String levStr = props.getProperty(Config.DEBUGLEVEL); - + if (levStr == null) { return Level.DEBUG; } - + try { int level = Integer.parseInt(levStr); if (level <= 0) { @@ -69,19 +69,19 @@ public static Level getDebugLevel(Properties props) if (lev != null) { return lev; } else { - throw new LinkBenchConfigError("Invalid setting for debug level: " + + throw new LinkBenchConfigError("Invalid setting for debug level: " + levStr); - } + } } } - + /** * Setup log4j to log to stderr with a timestamp and thread id * Could add in configuration from file later if it was really necessary - * @param props + * @param props * @param logFile if not null, info logging will be diverted to this file - * @throws IOException - * @throws Exception + * @throws IOException + * @throws Exception */ public static void setupLogging(Properties props, String logFile) throws LinkBenchConfigError, IOException { @@ -91,9 +91,9 @@ public static void setupLogging(Properties props, String logFile) Logger lbLogger = Logger.getLogger(LINKBENCH_LOGGER); lbLogger.setLevel(logLevel); ConsoleAppender console = new ConsoleAppender(fmt, "System.err"); - - /* If logfile is specified, put full stream in logfile and only - * print important messages to terminal + + /* If logfile is specified, put full stream in logfile and only + * print important messages to terminal */ if (logFile != null) { console.setThreshold(Level.WARN); // Only print salient messages @@ -101,7 +101,7 @@ public static void setupLogging(Properties props, String logFile) } lbLogger.addAppender(console); } - + /** * Look up key in props, failing if not present * @param props @@ -118,12 +118,12 @@ public static String getPropertyRequired(Properties props, String key) } return v; } - + public static int getInt(Properties props, String key) throws LinkBenchConfigError { return getInt(props, key, null); } - + /** * Retrieve a config key and convert to integer * @param props @@ -137,19 +137,19 @@ public static int getInt(Properties props, String key, Integer defaultVal) return defaultVal; } String v = getPropertyRequired(props, key); - try { + try { return Integer.parseInt(v); } catch (NumberFormatException e) { - throw new LinkBenchConfigError("Expected configuration key " + key + + throw new LinkBenchConfigError("Expected configuration key " + key + " to be integer, but was '" + v + "'"); } } - + public static long getLong(Properties props, String key) throws LinkBenchConfigError { - return getLong(props, key, null); + return getLong(props, key, null); } - + /** * Retrieve a config key and convert to long integer * @param props @@ -164,20 +164,20 @@ public static long getLong(Properties props, String key, Long defaultVal) return defaultVal; } String v = getPropertyRequired(props, key); - try { + try { return Long.parseLong(v); } catch (NumberFormatException e) { - throw new LinkBenchConfigError("Expected configuration key " + key + + throw new LinkBenchConfigError("Expected configuration key " + key + " to be long integer, but was '" + v + "'"); } } - - - public static double getDouble(Properties props, String key) + + + public static double getDouble(Properties props, String key) throws LinkBenchConfigError { return getDouble(props, key, null); } - + /** * Retrieve a config key and convert to double * @param props @@ -192,17 +192,17 @@ public static double getDouble(Properties props, String key, return defaultVal; } String v = getPropertyRequired(props, key); - try { + try { return Double.parseDouble(v); } catch (NumberFormatException e) { - throw new LinkBenchConfigError("Expected configuration key " + key + + throw new LinkBenchConfigError("Expected configuration key " + key + " to be double, but was '" + v + "'"); } } - + /** * Retrieve a config key and convert to boolean. - * Valid boolean strings are "true" or "false", case insensitive + * Valid boolean strings are "true" or "false", case insensitive * @param props * @param key * @return @@ -217,7 +217,7 @@ public static boolean getBool(Properties props, String key) } else if (v.equals("false")) { return false; } else { - throw new LinkBenchConfigError("Expected configuration key " + key + + throw new LinkBenchConfigError("Expected configuration key " + key + " to be true or false, but was '" + v + "'"); } } diff --git a/src/java/com/facebook/LinkBench/GraphStore.java b/src/java/com/facebook/LinkBench/GraphStore.java index cf9c27dd..827ad9e3 100644 --- a/src/java/com/facebook/LinkBench/GraphStore.java +++ b/src/java/com/facebook/LinkBench/GraphStore.java @@ -18,7 +18,7 @@ import java.util.List; /** - * An abstract class for storing both nodes and edges + * An abstract class for storing both nodes and edges * @author tarmstrong */ public abstract class GraphStore extends LinkStore implements NodeStore { diff --git a/src/java/com/facebook/LinkBench/InvertibleShuffler.java b/src/java/com/facebook/LinkBench/InvertibleShuffler.java index b69e5326..c41b774b 100644 --- a/src/java/com/facebook/LinkBench/InvertibleShuffler.java +++ b/src/java/com/facebook/LinkBench/InvertibleShuffler.java @@ -21,13 +21,13 @@ * Shuffler designed to make computing permutation and inverse easy */ public class InvertibleShuffler { - private final long[] params; + private final long[] params; private final int shuffleGroups; long n; long nRoundedUp; // n rounded up to next multiple of shuffleGroups long nRoundedDown; // n rounded down to next multiple of shuffleGroups int minGroupSize; - + public InvertibleShuffler(long seed, int shuffleGroups, long n) { this(new Random(seed), shuffleGroups, n); } @@ -40,23 +40,23 @@ public InvertibleShuffler(Random rng, int shuffleGroups, long n) { this.n = n; this.params = new long[shuffleGroups]; this.minGroupSize = (int)n / shuffleGroups; - + for (int i = 0; i < shuffleGroups; i++) { // Positive long params[i] = Math.abs(rng.nextInt(minGroupSize)); } - this.nRoundedDown = (n / shuffleGroups) * shuffleGroups; + this.nRoundedDown = (n / shuffleGroups) * shuffleGroups; this.nRoundedUp = n == nRoundedDown ? n : nRoundedDown + shuffleGroups; } - + public long permute(long i) { return permute(i, false); } - + public long invertPermute(long i) { return permute(i, true); } - + public long permute(long i, boolean inverse) { if (i < 0 || i >= n) { throw new IllegalArgumentException("Bad index to permute: " + i @@ -64,10 +64,10 @@ public long permute(long i, boolean inverse) { } // Number of the group int group = (int) (i % shuffleGroups); - + // Whether this is a big or small group boolean bigGroup = group < n % shuffleGroups; - + // Calculate the (positive) rotation long rotate = params[group]; if (inverse) { @@ -79,7 +79,7 @@ public long permute(long i, boolean inverse) { } assert(rotate >= 0); } - + long j = (i + shuffleGroups * rotate); long result; if (j < n) { diff --git a/src/java/com/facebook/LinkBench/Link.java b/src/java/com/facebook/LinkBench/Link.java index c2fdffe3..91e3a9c8 100644 --- a/src/java/com/facebook/LinkBench/Link.java +++ b/src/java/com/facebook/LinkBench/Link.java @@ -48,14 +48,14 @@ public boolean equals(Object other) { return false; } } - + public String toString() { return String.format("Link(id1=%d, id2=%d, link_type=%d," + - "visibility=%d, version=%d," + - "time=%d, data=%s", id1, id2, link_type, - visibility, version, time, data.toString()); + "visibility=%d, version=%d," + + "time=%d, data=%s", id1, id2, link_type, + visibility, version, time, data.toString()); } - + /** * Clone an existing link * @param l diff --git a/src/java/com/facebook/LinkBench/LinkBenchDriver.java b/src/java/com/facebook/LinkBench/LinkBenchDriver.java index ae3e3acb..ae692de2 100644 --- a/src/java/com/facebook/LinkBench/LinkBenchDriver.java +++ b/src/java/com/facebook/LinkBench/LinkBenchDriver.java @@ -60,10 +60,10 @@ Then does read and write requests of various types (addlink, deletelink, */ public class LinkBenchDriver { - + public static final int EXIT_BADARGS = 1; public static final int EXIT_BADCONFIG = 2; - + /* Command line arguments */ private static String configFile = null; private static String workloadConfigFile = null; @@ -75,10 +75,10 @@ public class LinkBenchDriver { private static PrintStream csvStreamFile = null; private static boolean doLoad = false; private static boolean doRequest = false; - + private Properties props; - - private final Logger logger = Logger.getLogger(ConfigUtil.LINKBENCH_LOGGER); + + private final Logger logger = Logger.getLogger(ConfigUtil.LINKBENCH_LOGGER); LinkBenchDriver(String configfile, Properties overrideProps, String logFile) @@ -89,9 +89,9 @@ public class LinkBenchDriver { for (String key: overrideProps.stringPropertyNames()) { props.setProperty(key, overrideProps.getProperty(key)); } - + loadWorkloadProps(); - + ConfigUtil.setupLogging(props, logFile); logger.info("Config file: " + configfile); @@ -146,7 +146,7 @@ private Stores initStores() throws Exception { LinkStore linkStore = createLinkStore(); NodeStore nodeStore = createNodeStore(linkStore); - + return new Stores(linkStore, nodeStore); } @@ -156,20 +156,20 @@ private LinkStore createLinkStore() throws Exception, IOException { // for easy access: // LinkStoreMysql : run benchmark on mySQL // LinkStoreHBaseGeneralAtomicityTesting : atomicity testing on HBase. - - String linkStoreClassName = ConfigUtil.getPropertyRequired(props, + + String linkStoreClassName = ConfigUtil.getPropertyRequired(props, Config.LINKSTORE_CLASS); - + logger.debug("Using LinkStore implementation: " + linkStoreClassName); - + LinkStore linkStore; try { - linkStore = ClassLoadUtil.newInstance(linkStoreClassName, + linkStore = ClassLoadUtil.newInstance(linkStoreClassName, LinkStore.class); } catch (ClassNotFoundException nfe) { throw new IOException("Cound not find class for " + linkStoreClassName); } - + return linkStore; } @@ -188,12 +188,12 @@ private NodeStore createNodeStore(LinkStore linkStore) throws Exception, } else { logger.debug("Using NodeStore implementation: " + nodeStoreClassName); } - + if (linkStore != null && linkStore.getClass().getName().equals( nodeStoreClassName)) { // Same class, reuse object if (!NodeStore.class.isAssignableFrom(linkStore.getClass())) { - throw new Exception("Specified NodeStore class " + nodeStoreClassName + throw new Exception("Specified NodeStore class " + nodeStoreClassName + " is not a subclass of NodeStore"); } return (NodeStore)linkStore; @@ -217,39 +217,39 @@ void load() throws IOException, InterruptedException, Throwable { } // load data int nLinkLoaders = ConfigUtil.getInt(props, Config.NUM_LOADERS); - + boolean bulkLoad = true; BlockingQueue chunk_q = new LinkedBlockingQueue(); - + // max id1 to generate long maxid1 = ConfigUtil.getLong(props, Config.MAX_ID); // id1 at which to start long startid1 = ConfigUtil.getLong(props, Config.MIN_ID); - + // Create loaders logger.info("Starting loaders " + nLinkLoaders); logger.debug("Bulk Load setting: " + bulkLoad); - + Random masterRandom = createMasterRNG(props, Config.LOAD_RANDOM_SEED); - + boolean genNodes = ConfigUtil.getBool(props, Config.GENERATE_NODES); int nTotalLoaders = genNodes ? nLinkLoaders + 1 : nLinkLoaders; - + LatencyStats latencyStats = new LatencyStats(nTotalLoaders); List loaders = new ArrayList(nTotalLoaders); - - LoadProgress loadTracker = LoadProgress.create(logger, props); + + LoadProgress loadTracker = LoadProgress.create(logger, props); for (int i = 0; i < nLinkLoaders; i++) { LinkStore linkStore = createLinkStore(); - + bulkLoad = bulkLoad && linkStore.bulkLoadBatchSize() > 0; - LinkBenchLoad l = new LinkBenchLoad(linkStore, props, latencyStats, + LinkBenchLoad l = new LinkBenchLoad(linkStore, props, latencyStats, csvStreamFile, i, maxid1 == startid1 + 1, chunk_q, loadTracker); loaders.add(l); } - + if (genNodes) { logger.info("Will generate graph nodes during loading"); int loaderId = nTotalLoaders - 1; @@ -258,7 +258,7 @@ void load() throws IOException, InterruptedException, Throwable { loaders.add(new NodeLoader(props, logger, nodeStore, rng, latencyStats, csvStreamFile, loaderId)); } - enqueueLoadWork(chunk_q, startid1, maxid1, nLinkLoaders, + enqueueLoadWork(chunk_q, startid1, maxid1, nLinkLoaders, new Random(masterRandom.nextLong())); // run loaders loadTracker.startTimer(); @@ -277,15 +277,15 @@ void load() throws IOException, InterruptedException, Throwable { } latencyStats.displayLatencyStats(); - + if (csvStatsFile != null) { latencyStats.printCSVStats(csvStatsFile, true); } - + double loadTime_s = (loadTime/1000.0); logger.info(String.format("LOAD PHASE COMPLETED. " + " Loaded %d nodes (Expected %d)." + - " Loaded %d links (%.2f links per node). " + + " Loaded %d links (%.2f links per node). " + " Took %.1f seconds. Links/second = %d", actualNodes, expectedNodes, actualLinks, actualLinks / (double) actualNodes, loadTime_s, @@ -307,22 +307,22 @@ private Random createMasterRNG(Properties props, String configKey) { logger.info("Using configured random seed " + configKey + "=" + seed); } else { seed = System.nanoTime() ^ (long)configKey.hashCode(); - logger.info("Using random seed " + seed + " since " + configKey + logger.info("Using random seed " + seed + " since " + configKey + " not specified"); } - + SecureRandom masterRandom; try { masterRandom = SecureRandom.getInstance("SHA1PRNG"); } catch (NoSuchAlgorithmException e) { logger.warn("SHA1PRNG not available, defaulting to default SecureRandom" + - " implementation"); + " implementation"); masterRandom = new SecureRandom(); } masterRandom.setSeed(ByteBuffer.allocate(8).putLong(seed).array()); - + // Can be used to check that rng is behaving as expected - logger.debug("First number generated by master " + configKey + + logger.debug("First number generated by master " + configKey + ": " + masterRandom.nextLong()); return masterRandom; } @@ -331,20 +331,20 @@ private void enqueueLoadWork(BlockingQueue chunk_q, long startid1, long maxid1, int nloaders, Random rng) { // Enqueue work chunks. Do it in reverse order as a heuristic to improve // load balancing, since queue is FIFO and later chunks tend to be larger - + int chunkSize = ConfigUtil.getInt(props, Config.LOADER_CHUNK_SIZE, 2048); long chunk_num = 0; ArrayList stack = new ArrayList(); for (long id1 = startid1; id1 < maxid1; id1 += chunkSize) { - stack.add(new LoadChunk(chunk_num, id1, + stack.add(new LoadChunk(chunk_num, id1, Math.min(id1 + chunkSize, maxid1), rng)); chunk_num++; } - + for (int i = stack.size() - 1; i >= 0; i--) { chunk_q.add(stack.get(i)); } - + for (int i = 0; i < nloaders; i++) { // Add a shutdown signal for each loader chunk_q.add(LoadChunk.SHUTDOWN); @@ -368,14 +368,14 @@ void sendrequests() throws IOException, InterruptedException, Throwable { List requesters = new LinkedList(); RequestProgress progress = LinkBenchRequest.createProgress(logger, props); - + Random masterRandom = createMasterRNG(props, Config.REQUEST_RANDOM_SEED); - + // create requesters for (int i = 0; i < nrequesters; i++) { Stores stores = initStores(); LinkBenchRequest l = new LinkBenchRequest(stores.linkStore, - stores.nodeStore, props, latencyStats, csvStreamFile, + stores.nodeStore, props, latencyStats, csvStreamFile, progress, new Random(masterRandom.nextLong()), i, nrequesters); requesters.add(l); } @@ -384,8 +384,8 @@ void sendrequests() throws IOException, InterruptedException, Throwable { concurrentExec(requesters); long finishTime = System.currentTimeMillis(); // Calculate duration accounting for warmup time - long benchmarkTime = finishTime - progress.getBenchmarkStartTime(); - + long benchmarkTime = finishTime - progress.getBenchmarkStartTime(); + long requestsdone = 0; int abortedRequesters = 0; // wait for requesters @@ -397,18 +397,18 @@ void sendrequests() throws IOException, InterruptedException, Throwable { } latencyStats.displayLatencyStats(); - + if (csvStatsFile != null) { latencyStats.printCSVStats(csvStatsFile, true); } - + logger.info("REQUEST PHASE COMPLETED. " + requestsdone + " requests done in " + (benchmarkTime/1000) + " seconds." + " Requests/second = " + (1000*requestsdone)/benchmarkTime); if (abortedRequesters > 0) { logger.error(String.format("Benchmark did not complete cleanly: %d/%d " + - "request threads aborted. See error log entries for details.", - abortedRequesters, nrequesters)); + "request threads aborted. See error log entries for details.", + abortedRequesters, nrequesters)); } } @@ -439,8 +439,7 @@ public void run() { task.run(); } catch (Throwable e) { Logger threadLog = Logger.getLogger(ConfigUtil.LINKBENCH_LOGGER); - threadLog.error("Unrecoverable exception in" + - " worker thread:", e); + threadLog.error("Unrecoverable exception in worker thread:", e); Runtime.getRuntime().halt(1); } doneSignal.countDown(); @@ -460,7 +459,7 @@ void drive() throws IOException, InterruptedException, Throwable { public static void main(String[] args) throws IOException, InterruptedException, Throwable { processArgs(args); - LinkBenchDriver d = new LinkBenchDriver(configFile, + LinkBenchDriver d = new LinkBenchDriver(configFile, cmdLineProps, logFile); try { d.drive(); @@ -481,26 +480,26 @@ private static Options initializeOptions() { Option config = new Option("c", true, "Linkbench config file"); config.setArgName("file"); options.addOption(config); - + Option log = new Option("L", true, "Log to this file"); log.setArgName("file"); options.addOption(log); - - Option csvStats = new Option("csvstats", "csvstats", true, + + Option csvStats = new Option("csvstats", "csvstats", true, "CSV stats output"); csvStats.setArgName("file"); options.addOption(csvStats); - - Option csvStream = new Option("csvstream", "csvstream", true, + + Option csvStream = new Option("csvstream", "csvstream", true, "CSV streaming stats output"); csvStream.setArgName("file"); options.addOption(csvStream); - + options.addOption("l", false, "Execute loading stage of benchmark"); options.addOption("r", false, "Execute request stage of benchmark"); - + // Java-style properties to override config file // -Dkey=value Option property = new Option("D", "Override a config setting"); @@ -508,10 +507,10 @@ private static Options initializeOptions() { property.setArgName("property=value"); property.setValueSeparator('='); options.addOption(property); - + return options; } - + /** * Process command line arguments and set static variables * exits program if invalid arguments provided @@ -522,7 +521,7 @@ private static Options initializeOptions() { private static void processArgs(String[] args) throws ParseException { Options options = initializeOptions(); - + CommandLine cmd = null; try { CommandLineParser parser = new GnuParser(); @@ -533,8 +532,8 @@ private static void processArgs(String[] args) printUsage(options); System.exit(EXIT_BADARGS); } - - /* + + /* * Apache CLI validates arguments, so can now assume * all required options are present, etc */ @@ -547,14 +546,14 @@ private static void processArgs(String[] args) System.err.println(); printUsage(options); System.exit(EXIT_BADARGS); - } + } // Set static option variables doLoad = cmd.hasOption('l'); doRequest = cmd.hasOption('r'); - + logFile = cmd.getOptionValue('L'); // May be null - + configFile = cmd.getOptionValue('c'); if (configFile == null) { // Try to find in usual location @@ -570,7 +569,7 @@ private static void processArgs(String[] args) System.exit(EXIT_BADARGS); } } - + String csvStatsFileName = cmd.getOptionValue("csvstats"); // May be null if (csvStatsFileName != null) { try { @@ -582,7 +581,7 @@ private static void processArgs(String[] args) System.exit(EXIT_BADARGS); } } - + String csvStreamFileName = cmd.getOptionValue("csvstream"); // May be null if (csvStreamFileName != null) { try { @@ -597,9 +596,9 @@ private static void processArgs(String[] args) System.exit(EXIT_BADARGS); } } - + cmdLineProps = cmd.getOptionProperties("D"); - + if (!(doLoad || doRequest)) { System.err.println("Did not select benchmark mode"); printUsage(options); diff --git a/src/java/com/facebook/LinkBench/LinkBenchDriverMR.java b/src/java/com/facebook/LinkBench/LinkBenchDriverMR.java index fd819257..980c0d39 100644 --- a/src/java/com/facebook/LinkBench/LinkBenchDriverMR.java +++ b/src/java/com/facebook/LinkBench/LinkBenchDriverMR.java @@ -72,7 +72,7 @@ public class LinkBenchDriverMR extends Configured implements Tool { private static final Logger logger = Logger.getLogger(ConfigUtil.LINKBENCH_LOGGER); - + static enum Counters { LINK_LOADED, REQUEST_DONE } private static Properties props; @@ -324,15 +324,15 @@ public void map(IntWritable loaderid, long maxid1 = ConfigUtil.getLong(props, Config.MAX_ID); long startid1 = ConfigUtil.getLong(props, Config.MIN_ID); - + LoadProgress prog_tracker = LoadProgress.create( Logger.getLogger(ConfigUtil.LINKBENCH_LOGGER), props); - - LinkBenchLoad loader = new LinkBenchLoad(store, props, latencyStats, + + LinkBenchLoad loader = new LinkBenchLoad(store, props, latencyStats, null, loaderid.get(), maxid1 == startid1 + 1, nloaders.get(), prog_tracker, new Random()); - + LinkedList tasks = new LinkedList(); tasks.add(loader); long linksloaded = 0; @@ -367,13 +367,13 @@ public void map(IntWritable requesterid, LatencyStats latencyStats = new LatencyStats(nrequesters.get()); RequestProgress progress = LinkBenchRequest.createProgress(logger, props); - progress.startTimer(); + progress.startTimer(); // TODO: Don't support NodeStore yet final LinkBenchRequest requester = new LinkBenchRequest(store, null, props, latencyStats, null, progress, new Random(), requesterid.get(), nrequesters.get()); - - + + // Wrap in runnable to handle error Thread t = new Thread(new Runnable() { public void run() { diff --git a/src/java/com/facebook/LinkBench/LinkBenchLoad.java b/src/java/com/facebook/LinkBench/LinkBenchLoad.java index 9fd8e05e..f1f32b45 100644 --- a/src/java/com/facebook/LinkBench/LinkBenchLoad.java +++ b/src/java/com/facebook/LinkBench/LinkBenchLoad.java @@ -41,7 +41,7 @@ * Multi-threaded loader for loading graph edges (but not nodes) into * LinkStore. The range from startid1 to maxid1 is chunked up into equal sized * disjoint ranges. These are then enqueued for processing by a number - * of loader threads to be loaded in parallel. The #links generated for + * of loader threads to be loaded in parallel. The #links generated for * an id1 is based on the configured distribution. The # of link types, * and link payload data is also controlled by the configuration file. * The actual counts of #links generated is tracked in nlinks_counts. @@ -49,7 +49,7 @@ public class LinkBenchLoad implements Runnable { - private final Logger logger = Logger.getLogger(ConfigUtil.LINKBENCH_LOGGER); + private final Logger logger = Logger.getLogger(ConfigUtil.LINKBENCH_LOGGER); private long maxid1; // max id1 to generate private long startid1; // id1 at which to start @@ -64,16 +64,16 @@ public class LinkBenchLoad implements Runnable { Level debuglevel; String dbid; - + private ID2Chooser id2chooser; // Counters for load statistics long sameShuffle; long diffShuffle; long linksloaded; - - /** - * special case for single hot row benchmark. If singleAssoc is set, + + /** + * special case for single hot row benchmark. If singleAssoc is set, * then make this method not print any statistics message, all statistics * are collected at a higher layer. */ boolean singleAssoc; @@ -103,13 +103,13 @@ public LinkBenchLoad(LinkStore store, Properties props, int nloaders, LoadProgress prog_tracker, Random rng) { this(store, props, latencyStats, csvStreamOut, loaderID, singleAssoc, new ArrayBlockingQueue(2), prog_tracker); - + // Just add a single chunk to the queue chunk_q.add(new LoadChunk(loaderID, startid1, maxid1, rng)); chunk_q.add(LoadChunk.SHUTDOWN); } - + public LinkBenchLoad(LinkStore linkStore, Properties props, LatencyStats latencyStats, @@ -128,27 +128,27 @@ public LinkBenchLoad(LinkStore linkStore, this.singleAssoc = singleAssoc; this.chunk_q = chunk_q; this.prog_tracker = prog_tracker; - - + + /* * Load settings from properties */ maxid1 = ConfigUtil.getLong(props, Config.MAX_ID); startid1 = ConfigUtil.getLong(props, Config.MIN_ID); - + // math functions may cause problems for id1 = 0. Start at 1. if (startid1 <= 0) { throw new LinkBenchConfigError("startid1 must be >= 1"); } debuglevel = ConfigUtil.getDebugLevel(props); - + double medianLinkDataSize = ConfigUtil.getDouble(props, Config.LINK_DATASIZE); linkDataSize = new LogNormalDistribution(); linkDataSize.init(0, LinkStore.MAX_LINK_DATA, medianLinkDataSize, Config.LINK_DATASIZE_SIGMA); - + try { linkDataGen = ClassLoadUtil.newInstance( ConfigUtil.getPropertyRequired(props, Config.LINK_ADD_DATAGEN), @@ -156,15 +156,15 @@ public LinkBenchLoad(LinkStore linkStore, linkDataGen.init(props, Config.LINK_ADD_DATAGEN_PREFIX); } catch (ClassNotFoundException ex) { logger.error(ex); - throw new LinkBenchConfigError("Error loading data generator class: " + throw new LinkBenchConfigError("Error loading data generator class: " + ex.getMessage()); } - + displayFreq_ms = ConfigUtil.getLong(props, Config.DISPLAY_FREQ) * 1000; int maxsamples = ConfigUtil.getInt(props, Config.MAX_STAT_SAMPLES); - + dbid = ConfigUtil.getPropertyRequired(props, Config.DBID); - + /* * Initialize statistics */ @@ -172,7 +172,7 @@ public LinkBenchLoad(LinkStore linkStore, sameShuffle = 0; diffShuffle = 0; stats = new SampledStats(loaderID, maxsamples, csvStreamOut); - + id2chooser = new ID2Chooser(props, startid1, maxid1, 1, 1); } @@ -188,7 +188,7 @@ public void run() { logger.error("Error while initializing store", e); throw new RuntimeException(e); } - + int bulkLoadBatchSize = store.bulkLoadBatchSize(); boolean bulkLoad = bulkLoadBatchSize > 0; ArrayList loadBuffer = null; @@ -200,17 +200,17 @@ public void run() { logger.info("Starting loader thread #" + loaderID + " loading links"); lastDisplayTime = System.currentTimeMillis(); - + while (true) { LoadChunk chunk; try { chunk = chunk_q.take(); - //logger.info("chunk end="+chunk.end); + //logger.info("chunk end="+chunk.end); } catch (InterruptedException ie) { logger.warn("InterruptedException not expected, try again", ie); continue; } - + // Shutdown signal is received though special chunk type if (chunk.shutdown) { break; @@ -220,19 +220,19 @@ public void run() { processChunk(chunk, bulkLoad, bulkLoadBatchSize, loadBuffer, countLoadBuffer); } - + if (bulkLoad) { // Load any remaining links or counts loadLinks(loadBuffer); loadCounts(countLoadBuffer); } - + if (!singleAssoc) { logger.debug(" Same shuffle = " + sameShuffle + " Different shuffle = " + diffShuffle); displayStats(lastDisplayTime, bulkLoad); } - + store.close(); } @@ -240,7 +240,7 @@ public void run() { private void displayStats(long startTime, boolean bulkLoad) { long endTime = System.currentTimeMillis(); if (bulkLoad) { - stats.displayStats(startTime, endTime, + stats.displayStats(startTime, endTime, Arrays.asList(LinkBenchOp.LOAD_LINKS_BULK, LinkBenchOp.LOAD_COUNTS_BULK, LinkBenchOp.LOAD_LINKS_BULK_NLINKS, LinkBenchOp.LOAD_COUNTS_BULK_NLINKS)); @@ -257,7 +257,7 @@ private void processChunk(LoadChunk chunk, boolean bulkLoad, logger.debug("Loader thread #" + loaderID + " processing " + chunk.toString()); } - + // Counter for total number of links loaded in chunk; long links_in_chunk = 0; @@ -267,13 +267,13 @@ private void processChunk(LoadChunk chunk, boolean bulkLoad, // otherwise reuse object link = initLink(); } - + long prevPercentPrinted = 0; for (long id1 = chunk.start; id1 < chunk.end; id1 += chunk.step) { long added_links= createOutLinks(chunk.rng, link, loadBuffer, countLoadBuffer, id1, singleAssoc, bulkLoad, bulkLoadBatchSize); links_in_chunk += added_links; - + if (!singleAssoc) { long nloaded = (id1 - chunk.start) / chunk.step; if (bulkLoad) { @@ -285,7 +285,7 @@ private void processChunk(LoadChunk chunk, boolean bulkLoad, prevPercentPrinted = percent; } } - + // Check if stats should be flushed and reset long now = System.currentTimeMillis(); if (lastDisplayTime + displayFreq_ms <= now) { @@ -294,7 +294,7 @@ private void processChunk(LoadChunk chunk, boolean bulkLoad, lastDisplayTime = now; } } - + // Update progress and maybe print message prog_tracker.update(chunk.size, links_in_chunk); } @@ -319,7 +319,7 @@ private long createOutLinks(Random rng, linkTypeCounts = new HashMap(); } long nlinks_total = 0; - + for (long link_type: id2chooser.getLinkTypes()) { long nlinks = id2chooser.calcLinkCount(id1, link_type); nlinks_total += nlinks; @@ -328,7 +328,7 @@ private long createOutLinks(Random rng, } else { diffShuffle++; } - + if (Level.TRACE.isGreaterOrEqual(debuglevel)) { logger.trace("id1 = " + id1 + " link_type = " + link_type + " nlinks = " + nlinks); @@ -339,15 +339,15 @@ private long createOutLinks(Random rng, link = initLink(); } constructLink(rng, link, id1, link_type, j, singleAssoc); - + if (bulkLoad) { loadBuffer.add(link); if (loadBuffer.size() >= bulkLoadBatchSize) { loadLinks(loadBuffer); } - + // Update link counts for this type - LinkCount count = linkTypeCounts.get(link.link_type); + LinkCount count = linkTypeCounts.get(link.link_type); if (count == null) { count = new LinkCount(id1, link.link_type, link.time, link.version, 1); @@ -361,7 +361,7 @@ private long createOutLinks(Random rng, loadLink(link, j, nlinks, singleAssoc); } } - + } // Maintain the counts separately @@ -398,7 +398,7 @@ private void constructLink(Random rng, Link link, long id1, long link_type, long outlink_ix, boolean singleAssoc) { link.id1 = id1; link.link_type = link_type; - + // Using random number generator for id2 means we won't know // which id2s exist. So link id1 to // maxid1 + id1 + 1 thru maxid1 + id1 + nlinks(id1) UNLESS @@ -428,7 +428,7 @@ private long chooseInitialTimestamp(Random rng) { /** * Load an individual link into the db. - * + * * If an error occurs during loading, this method will log it, * add stats, and reset the connection. * @param link @@ -442,22 +442,22 @@ private void loadLink(Link link, long outlink_ix, long nlinks, if (!singleAssoc) { timestart = System.nanoTime(); } - + try { // no inverses for now store.addLink(dbid, link, true); linksloaded++; - + if (!singleAssoc && outlink_ix == nlinks - 1) { long timetaken = (System.nanoTime() - timestart); - + // convert to microseconds stats.addStats(LinkBenchOp.LOAD_LINK, timetaken/1000, false); - - latencyStats.recordLatency(loaderID, + + latencyStats.recordLatency(loaderID, LinkBenchOp.LOAD_LINK, timetaken); } - + } catch (Throwable e){//Catch exception if any long endtime2 = System.nanoTime(); long timetaken2 = (endtime2 - timestart)/1000; @@ -475,13 +475,13 @@ private void loadLinks(ArrayList loadBuffer) { store.addBulkLinks(dbid, loadBuffer, true); linksloaded += nlinks; loadBuffer.clear(); - + long timetaken = (System.nanoTime() - timestart); - + // convert to microseconds stats.addStats(LinkBenchOp.LOAD_LINKS_BULK, timetaken/1000, false); stats.addStats(LinkBenchOp.LOAD_LINKS_BULK_NLINKS, nlinks, false); - + latencyStats.recordLatency(loaderID, LinkBenchOp.LOAD_LINKS_BULK, timetaken); } catch (Throwable e){//Catch exception if any @@ -492,22 +492,22 @@ private void loadLinks(ArrayList loadBuffer) { store.clearErrors(loaderID); } } - + private void loadCounts(ArrayList loadBuffer) { long timestart = System.nanoTime(); - + try { // no inverses for now int ncounts = loadBuffer.size(); store.addBulkCounts(dbid, loadBuffer); loadBuffer.clear(); - + long timetaken = (System.nanoTime() - timestart); - + // convert to microseconds stats.addStats(LinkBenchOp.LOAD_COUNTS_BULK, timetaken/1000, false); stats.addStats(LinkBenchOp.LOAD_COUNTS_BULK_NLINKS, ncounts, false); - + latencyStats.recordLatency(loaderID, LinkBenchOp.LOAD_COUNTS_BULK, timetaken); } catch (Throwable e){//Catch exception if any @@ -518,7 +518,7 @@ private void loadCounts(ArrayList loadBuffer) { store.clearErrors(loaderID); } } - + /** * Represents a portion of the id space, starting with * start, going up until end (non-inclusive) with step size @@ -550,7 +550,7 @@ public LoadChunk(boolean shutdown, public final long step; public final long size; public Random rng; - + public String toString() { if (shutdown) { return "chunk SHUTDOWN"; @@ -567,7 +567,7 @@ public String toString() { public static class LoadProgress { /** report progress at intervals of progressReportInterval links */ private final long progressReportInterval; - + public LoadProgress(Logger progressLogger, long id1s_total, long progressReportInterval) { super(); @@ -578,27 +578,27 @@ public LoadProgress(Logger progressLogger, this.id1s_loaded = new AtomicLong(); this.links_loaded = new AtomicLong(); } - + public static LoadProgress create(Logger progressLogger, Properties props) { long maxid1 = ConfigUtil.getLong(props, Config.MAX_ID); long startid1 = ConfigUtil.getLong(props, Config.MIN_ID); long nids = maxid1 - startid1; - long progressReportInterval = ConfigUtil.getLong(props, + long progressReportInterval = ConfigUtil.getLong(props, Config.LOAD_PROG_INTERVAL, 50000L); return new LoadProgress(progressLogger, nids, progressReportInterval); } - + private final Logger progressLogger; private final AtomicLong id1s_loaded; // progress private final AtomicLong links_loaded; // progress private final long id1s_total; // goal private long starttime_ms; - + /** Mark current time as start time for load */ public void startTimer() { starttime_ms = System.currentTimeMillis(); } - + /** * Update progress * @param id1_incr number of additional id1s loaded since last call @@ -606,10 +606,10 @@ public void startTimer() { */ public void update(long id1_incr, long links_incr) { long curr_id1s = id1s_loaded.addAndGet(id1_incr); - + long curr_links = links_loaded.addAndGet(links_incr); long prev_links = curr_links - links_incr; - + if ((curr_links / progressReportInterval) > (prev_links / progressReportInterval) || curr_id1s == id1s_total) { double percentage = (curr_id1s / (double)id1s_total) * 100.0; @@ -620,7 +620,7 @@ public void update(long id1_incr, long links_incr) { double id1_rate = ((curr_id1s) / ((double) now - starttime_ms))*1000; progressLogger.info(String.format( "%d/%d id1s loaded (%.1f%% complete) at %.2f id1s/sec avg. " + - "%d links loaded at %.2f links/sec avg.", + "%d links loaded at %.2f links/sec avg.", curr_id1s, id1s_total, percentage, id1_rate, curr_links, link_rate)); } diff --git a/src/java/com/facebook/LinkBench/LinkBenchOp.java b/src/java/com/facebook/LinkBench/LinkBenchOp.java index 89436b5c..7854bcfa 100644 --- a/src/java/com/facebook/LinkBench/LinkBenchOp.java +++ b/src/java/com/facebook/LinkBench/LinkBenchOp.java @@ -32,13 +32,13 @@ public enum LinkBenchOp { LOAD_LINKS_BULK, LOAD_COUNTS_BULK, // Although the following are not truly operations, we need stats - // for them + // for them RANGE_SIZE, // how big range scans are LOAD_LINKS_BULK_NLINKS, // how many links inserted in bulk LOAD_COUNTS_BULK_NLINKS, // how many counts inserted in bulk UNKNOWN; - + public String displayName() { return name(); } -} \ No newline at end of file +} diff --git a/src/java/com/facebook/LinkBench/LinkBenchRequest.java b/src/java/com/facebook/LinkBench/LinkBenchRequest.java index f05f9d17..606dc980 100644 --- a/src/java/com/facebook/LinkBench/LinkBenchRequest.java +++ b/src/java/com/facebook/LinkBench/LinkBenchRequest.java @@ -43,22 +43,22 @@ public class LinkBenchRequest implements Runnable { Properties props; LinkStore linkStore; NodeStore nodeStore; - + RequestProgress progressTracker; long numRequests; /** Requests per second: <= 0 for unlimited rate */ private long requestrate; - + /** Maximum number of failed requests: < 0 for unlimited */ private long maxFailedRequests; - - /** + + /** * Time to run benchmark for before collecting stats. Allows * caches, etc to warm up. */ private long warmupTime; - + /** Maximum time to run benchmark for, not including warmup time */ long maxTime; int nrequesters; @@ -78,7 +78,7 @@ public class LinkBenchRequest implements Runnable { private LogNormalDistribution nodeDataSize; private DataGenerator nodeAddDataGen; private DataGenerator nodeUpDataGen; - + // cummulative percentages double pc_addlink; double pc_deletelink; @@ -90,10 +90,10 @@ public class LinkBenchRequest implements Runnable { double pc_deletenode; double pc_updatenode; double pc_getnode; - + // Chance of doing historical range query double p_historical_getlinklist; - + private static class HistoryKey { public final long id1; public final long link_type; @@ -102,11 +102,11 @@ public HistoryKey(long id1, long link_type) { this.id1 = id1; this.link_type = link_type; } - + public HistoryKey(Link l) { this(l.id1, l.link_type); } - + @Override public int hashCode() { final int prime = 31; @@ -115,7 +115,7 @@ public int hashCode() { result = prime * result + (int) (link_type ^ (link_type >>> 32)); return result; } - + @Override public boolean equals(Object obj) { if (!(obj instanceof HistoryKey)) @@ -123,21 +123,21 @@ public boolean equals(Object obj) { HistoryKey other = (HistoryKey) obj; return id1 == other.id1 && link_type == other.link_type; } - + } - + // Cache of last link in lists where full list wasn't retrieved ArrayList listTailHistory; - + // Index of history to avoid duplicates HashMap listTailHistoryIndex; - + // Limit of cache size private int listTailHistoryLimit; - + // Probability distribution for ids in multiget ProbabilityDistribution multigetDist; - + // Statistics SampledStats stats; LatencyStats latencyStats; @@ -146,17 +146,17 @@ public boolean equals(Object obj) { long numfound = 0; long numnotfound = 0; long numHistoryQueries = 0; - - /** + + /** * Random number generator use for generating workload. If * initialized with same seed, should generate same sequence of requests - * so that tests and benchmarks are repeatable. + * so that tests and benchmarks are repeatable. */ Random rng; - + // Last node id accessed long lastNodeId; - + long requestsDone = 0; long errors = 0; boolean aborted; @@ -171,7 +171,7 @@ public boolean equals(Object obj) { private AccessDistribution nodeReadDist; // node reads private AccessDistribution nodeUpdateDist; // node writes private AccessDistribution nodeDeleteDist; // node deletes - + private ID2Chooser id2chooser; public LinkBenchRequest(LinkStore linkStore, NodeStore nodeStore, @@ -184,10 +184,10 @@ public LinkBenchRequest(LinkStore linkStore, int nrequesters) { assert(linkStore != null); if (requesterID < 0 || requesterID >= nrequesters) { - throw new IllegalArgumentException("Bad requester id " + throw new IllegalArgumentException("Bad requester id " + requesterID + "/" + nrequesters); } - + this.linkStore = linkStore; this.nodeStore = nodeStore; this.props = props; @@ -238,13 +238,13 @@ public LinkBenchRequest(LinkStore linkStore, progressFreq_ms = ConfigUtil.getLong(props, Config.PROGRESS_FREQ, 6L) * 1000; int maxsamples = ConfigUtil.getInt(props, Config.MAX_STAT_SAMPLES); stats = new SampledStats(requesterID, maxsamples, csvStreamOut); - + listTailHistoryLimit = 2048; // Hardcoded limit for now listTailHistory = new ArrayList(listTailHistoryLimit); listTailHistoryIndex = new HashMap(); p_historical_getlinklist = ConfigUtil.getDouble(props, - Config.PR_GETLINKLIST_HISTORY, 0.0) / 100; - + Config.PR_GETLINKLIST_HISTORY, 0.0) / 100; + lastNodeId = startid1; } @@ -255,49 +255,49 @@ private void initRequestProbabilities(Properties props) { pc_countlink = pc_updatelink + ConfigUtil.getDouble(props, Config.PR_COUNT_LINKS); pc_getlink = pc_countlink + ConfigUtil.getDouble(props, Config.PR_GET_LINK); pc_getlinklist = pc_getlink + ConfigUtil.getDouble(props, Config.PR_GET_LINK_LIST); - + pc_addnode = pc_getlinklist + ConfigUtil.getDouble(props, Config.PR_ADD_NODE, 0.0); pc_updatenode = pc_addnode + ConfigUtil.getDouble(props, Config.PR_UPDATE_NODE, 0.0); pc_deletenode = pc_updatenode + ConfigUtil.getDouble(props, Config.PR_DELETE_NODE, 0.0); pc_getnode = pc_deletenode + ConfigUtil.getDouble(props, Config.PR_GET_NODE, 0.0); - + if (Math.abs(pc_getnode - 100.0) > 1e-5) {//compare real numbers - throw new LinkBenchConfigError("Percentages of request types do not " + + throw new LinkBenchConfigError("Percentages of request types do not " + "add to 100, only " + pc_getnode + "!"); } } private void initLinkRequestDistributions(Properties props, int requesterID, int nrequesters) { - writeDist = AccessDistributions.loadAccessDistribution(props, + writeDist = AccessDistributions.loadAccessDistribution(props, startid1, maxid1, DistributionType.LINK_WRITES); - readDist = AccessDistributions.loadAccessDistribution(props, + readDist = AccessDistributions.loadAccessDistribution(props, startid1, maxid1, DistributionType.LINK_READS); - + // Load uncorrelated distributions for blending if needed writeDistUncorr = null; if (props.containsKey(Config.WRITE_UNCORR_BLEND)) { // Ratio of queries to use uncorrelated. Convert from percentage - writeDistUncorrBlend = ConfigUtil.getDouble(props, + writeDistUncorrBlend = ConfigUtil.getDouble(props, Config.WRITE_UNCORR_BLEND) / 100.0; if (writeDistUncorrBlend > 0.0) { - writeDistUncorr = AccessDistributions.loadAccessDistribution(props, + writeDistUncorr = AccessDistributions.loadAccessDistribution(props, startid1, maxid1, DistributionType.LINK_WRITES_UNCORR); } } - + readDistUncorr = null; if (props.containsKey(Config.READ_UNCORR_BLEND)) { // Ratio of queries to use uncorrelated. Convert from percentage - readDistUncorrBlend = ConfigUtil.getDouble(props, + readDistUncorrBlend = ConfigUtil.getDouble(props, Config.READ_UNCORR_BLEND) / 100.0; if (readDistUncorrBlend > 0.0) { - readDistUncorr = AccessDistributions.loadAccessDistribution(props, + readDistUncorr = AccessDistributions.loadAccessDistribution(props, startid1, maxid1, DistributionType.LINK_READS_UNCORR); } } - - id2chooser = new ID2Chooser(props, startid1, maxid1, + + id2chooser = new ID2Chooser(props, startid1, maxid1, nrequesters, requesterID); // Distribution of #id2s per multiget @@ -308,11 +308,11 @@ private void initLinkRequestDistributions(Properties props, int requesterID, try { multigetDist = ClassLoadUtil.newInstance(multigetDistClass, ProbabilityDistribution.class); - multigetDist.init(multigetMin, multigetMax, props, + multigetDist.init(multigetMin, multigetMax, props, Config.LINK_MULTIGET_DIST_PREFIX); } catch (ClassNotFoundException e) { logger.error(e); - throw new LinkBenchConfigError("Class" + multigetDistClass + + throw new LinkBenchConfigError("Class" + multigetDistClass + " could not be loaded as ProbabilityDistribution"); } } else { @@ -322,7 +322,7 @@ private void initLinkRequestDistributions(Properties props, int requesterID, private void initLinkDataGeneration(Properties props) { try { - double medLinkDataSize = ConfigUtil.getDouble(props, + double medLinkDataSize = ConfigUtil.getDouble(props, Config.LINK_DATASIZE); linkDataSize = new LogNormalDistribution(); linkDataSize.init(0, LinkStore.MAX_LINK_DATA, medLinkDataSize, @@ -331,21 +331,21 @@ private void initLinkDataGeneration(Properties props) { ConfigUtil.getPropertyRequired(props, Config.LINK_ADD_DATAGEN), DataGenerator.class); linkAddDataGen.init(props, Config.LINK_ADD_DATAGEN_PREFIX); - + linkUpDataGen = ClassLoadUtil.newInstance( ConfigUtil.getPropertyRequired(props, Config.LINK_UP_DATAGEN), DataGenerator.class); linkUpDataGen.init(props, Config.LINK_UP_DATAGEN_PREFIX); } catch (ClassNotFoundException ex) { logger.error(ex); - throw new LinkBenchConfigError("Error loading data generator class: " + throw new LinkBenchConfigError("Error loading data generator class: " + ex.getMessage()); } } private void initNodeRequestDistributions(Properties props) { try { - nodeReadDist = AccessDistributions.loadAccessDistribution(props, + nodeReadDist = AccessDistributions.loadAccessDistribution(props, startid1, maxid1, DistributionType.NODE_READS); } catch (LinkBenchConfigError e) { // Not defined @@ -354,9 +354,9 @@ private void initNodeRequestDistributions(Properties props) { throw new LinkBenchConfigError("Node read distribution not " + "configured but node read operations have non-zero probability"); } - + try { - nodeUpdateDist = AccessDistributions.loadAccessDistribution(props, + nodeUpdateDist = AccessDistributions.loadAccessDistribution(props, startid1, maxid1, DistributionType.NODE_UPDATES); } catch (LinkBenchConfigError e) { // Not defined @@ -365,9 +365,9 @@ private void initNodeRequestDistributions(Properties props) { throw new LinkBenchConfigError("Node write distribution not " + "configured but node write operations have non-zero probability"); } - + try { - nodeDeleteDist = AccessDistributions.loadAccessDistribution(props, + nodeDeleteDist = AccessDistributions.loadAccessDistribution(props, startid1, maxid1, DistributionType.NODE_DELETES); } catch (LinkBenchConfigError e) { // Not defined @@ -379,27 +379,27 @@ private void initNodeRequestDistributions(Properties props) { } private void initNodeDataGeneration(Properties props) { - try { - double medNodeDataSize = ConfigUtil.getDouble(props, + try { + double medNodeDataSize = ConfigUtil.getDouble(props, Config.NODE_DATASIZE); nodeDataSize = new LogNormalDistribution(); nodeDataSize.init(0, NodeStore.MAX_NODE_DATA, medNodeDataSize, Config.NODE_DATASIZE_SIGMA); - String dataGenClass = ConfigUtil.getPropertyRequired(props, + String dataGenClass = ConfigUtil.getPropertyRequired(props, Config.NODE_ADD_DATAGEN); nodeAddDataGen = ClassLoadUtil.newInstance(dataGenClass, DataGenerator.class); nodeAddDataGen.init(props, Config.NODE_ADD_DATAGEN_PREFIX); - - dataGenClass = ConfigUtil.getPropertyRequired(props, + + dataGenClass = ConfigUtil.getPropertyRequired(props, Config.NODE_UP_DATAGEN); nodeUpDataGen = ClassLoadUtil.newInstance(dataGenClass, DataGenerator.class); nodeUpDataGen.init(props, Config.NODE_UP_DATAGEN_PREFIX); } catch (ClassNotFoundException ex) { logger.error(ex); - throw new LinkBenchConfigError("Error loading data generator class: " + throw new LinkBenchConfigError("Error loading data generator class: " + ex.getMessage()); } } @@ -407,7 +407,7 @@ private void initNodeDataGeneration(Properties props) { public long getRequestsDone() { return requestsDone; } - + public boolean didAbort() { return aborted; } @@ -455,7 +455,7 @@ private long chooseRequestID(DistributionType type, long previousId1) { " for access distribution: " + dist.getClass().getName() + ": " + dist.toString()); } - + if (dist.getShuffler() != null) { // Shuffle to go from position in space ranked from most to least accessed, // to the real id space @@ -490,7 +490,7 @@ private boolean oneRequest(boolean recordStats) { link.visibility = LinkStore.VISIBILITY_DEFAULT; link.version = 0; link.time = System.currentTimeMillis(); - link.data = linkAddDataGen.fill(rng, + link.data = linkAddDataGen.fill(rng, new byte[(int)linkDataSize.choose(rng)]); starttime = System.nanoTime(); @@ -499,9 +499,9 @@ private boolean oneRequest(boolean recordStats) { boolean added = !alreadyExists; endtime = System.nanoTime(); if (Level.TRACE.isGreaterOrEqual(debuglevel)) { - logger.trace("addLink id1=" + link.id1 + " link_type=" + logger.trace("addLink id1=" + link.id1 + " link_type=" + link.link_type + " id2=" + link.id2 + " added=" + added); - } + } } else if (r <= pc_deletelink) { type = LinkBenchOp.DELETE_LINK; long id1 = chooseRequestID(DistributionType.LINK_WRITES, link.id1); @@ -513,9 +513,9 @@ private boolean oneRequest(boolean recordStats) { false); endtime = System.nanoTime(); if (Level.TRACE.isGreaterOrEqual(debuglevel)) { - logger.trace("deleteLink id1=" + id1 + " link_type=" + link_type + logger.trace("deleteLink id1=" + id1 + " link_type=" + link_type + " id2=" + id2); - } + } } else if (r <= pc_updatelink) { type = LinkBenchOp.UPDATE_LINK; link.id1 = chooseRequestID(DistributionType.LINK_WRITES, link.id1); @@ -526,8 +526,8 @@ private boolean oneRequest(boolean recordStats) { link.visibility = LinkStore.VISIBILITY_DEFAULT; link.version = 0; link.time = System.currentTimeMillis(); - link.data = linkUpDataGen.fill(rng, - new byte[(int)linkDataSize.choose(rng)]); + link.data = linkUpDataGen.fill(rng, + new byte[(int)linkDataSize.choose(rng)]); starttime = System.nanoTime(); // no inverses for now @@ -535,9 +535,9 @@ private boolean oneRequest(boolean recordStats) { boolean found = found1; endtime = System.nanoTime(); if (Level.TRACE.isGreaterOrEqual(debuglevel)) { - logger.trace("updateLink id1=" + link.id1 + " link_type=" + logger.trace("updateLink id1=" + link.id1 + " link_type=" + link.link_type + " id2=" + link.id2 + " found=" + found); - } + } } else if (r <= pc_countlink) { type = LinkBenchOp.COUNT_LINK; @@ -548,9 +548,9 @@ private boolean oneRequest(boolean recordStats) { long count = linkStore.countLinks(dbid, id1, link_type); endtime = System.nanoTime(); if (Level.TRACE.isGreaterOrEqual(debuglevel)) { - logger.trace("countLink id1=" + id1 + " link_type=" + link_type + logger.trace("countLink id1=" + id1 + " link_type=" + link_type + " count=" + count); - } + } } else if (r <= pc_getlink) { type = LinkBenchOp.MULTIGET_LINK; @@ -558,7 +558,7 @@ private boolean oneRequest(boolean recordStats) { long id1 = chooseRequestID(DistributionType.LINK_READS, link.id1); long link_type = id2chooser.chooseRandomLinkType(rng); int nid2s = 1; - if (multigetDist != null) { + if (multigetDist != null) { nid2s = (int)multigetDist.choose(rng); } long id2s[] = id2chooser.chooseMultipleForOp(rng, id1, link_type, nid2s, @@ -579,7 +579,7 @@ private boolean oneRequest(boolean recordStats) { type = LinkBenchOp.GET_LINKS_LIST; Link links[]; - + if (rng.nextDouble() < p_historical_getlinklist && !this.listTailHistory.isEmpty()) { links = getLinkListTail(); @@ -590,7 +590,7 @@ private boolean oneRequest(boolean recordStats) { links = getLinkList(id1, link_type); endtime = System.nanoTime(); } - + int count = ((links == null) ? 0 : links.length); if (recordStats) { stats.addStats(LinkBenchOp.RANGE_SIZE, count, false); @@ -608,21 +608,21 @@ private boolean oneRequest(boolean recordStats) { type = LinkBenchOp.UPDATE_NODE; // Choose an id that has previously been created (but might have // been since deleted - long upId = chooseRequestID(DistributionType.NODE_UPDATES, + long upId = chooseRequestID(DistributionType.NODE_UPDATES, lastNodeId); // Generate new data randomly Node newNode = createUpdateNode(upId); - + starttime = System.nanoTime(); boolean changed = nodeStore.updateNode(dbid, newNode); endtime = System.nanoTime(); lastNodeId = upId; if (Level.TRACE.isGreaterOrEqual(debuglevel)) { logger.trace("updateNode " + newNode + " changed=" + changed); - } + } } else if (r <= pc_deletenode) { type = LinkBenchOp.DELETE_NODE; - long idToDelete = chooseRequestID(DistributionType.NODE_DELETES, + long idToDelete = chooseRequestID(DistributionType.NODE_DELETES, lastNodeId); starttime = System.nanoTime(); boolean deleted = nodeStore.deleteNode(dbid, LinkStore.DEFAULT_NODE_TYPE, @@ -635,7 +635,7 @@ private boolean oneRequest(boolean recordStats) { } else if (r <= pc_getnode) { type = LinkBenchOp.GET_NODE; starttime = System.nanoTime(); - long idToFetch = chooseRequestID(DistributionType.NODE_READS, + long idToFetch = chooseRequestID(DistributionType.NODE_READS, lastNodeId); Node fetched = nodeStore.getNode(dbid, LinkStore.DEFAULT_NODE_TYPE, idToFetch); endtime = System.nanoTime(); @@ -661,7 +661,7 @@ private boolean oneRequest(boolean recordStats) { stats.addStats(type, timetaken, false); latencyStats.recordLatency(requesterID, type, timetaken); } - + return true; } catch (Throwable e){//Catch exception if any @@ -685,16 +685,16 @@ private boolean oneRequest(boolean recordStats) { */ private Node createAddNode() { byte data[] = nodeAddDataGen.fill(rng, new byte[(int)nodeDataSize.choose(rng)]); - return new Node(-1, LinkStore.DEFAULT_NODE_TYPE, 1, + return new Node(-1, LinkStore.DEFAULT_NODE_TYPE, 1, (int)(System.currentTimeMillis()/1000), data); } - + /** * Create new node for updating in database */ private Node createUpdateNode(long id) { byte data[] = nodeUpDataGen.fill(rng, new byte[(int)nodeDataSize.choose(rng)]); - return new Node(id, LinkStore.DEFAULT_NODE_TYPE, 2, + return new Node(id, LinkStore.DEFAULT_NODE_TYPE, 2, (int)(System.currentTimeMillis()/1000), data); } @@ -704,7 +704,7 @@ public void run() { + numRequests + " ops after " + warmupTime + " second warmup"); logger.debug("Requester thread #" + requesterID + " first random number " + rng.nextLong()); - + try { this.linkStore.initialize(props, Phase.REQUEST, requesterID); if (this.nodeStore != null && this.nodeStore != this.linkStore) { @@ -714,10 +714,10 @@ public void run() { logger.error("Error while initializing store", e); throw new RuntimeException(e); } - + long warmupStartTime = System.currentTimeMillis(); boolean warmupDone = warmupTime <= 0; - long benchmarkStartTime; + long benchmarkStartTime; if (!warmupDone) { benchmarkStartTime = warmupStartTime + warmupTime * 1000; } else { @@ -726,7 +726,7 @@ public void run() { long endTime = benchmarkStartTime + maxTime * 1000; long lastUpdate = warmupStartTime; long curTime = warmupStartTime; - + long i; if (singleAssoc) { @@ -760,7 +760,7 @@ public void run() { closeStores(); return; } - + long warmupRequests = 0; long requestsSinceLastUpdate = 0; long lastStatDisplay_ms = curTime; @@ -775,14 +775,14 @@ public void run() { errors++; if (maxFailedRequests >= 0 && errors > maxFailedRequests) { logger.error(String.format("Requester #%d aborting: %d failed requests" + - " (out of %d total) ", requesterID, errors, requestsDone)); + " (out of %d total) ", requesterID, errors, requestsDone)); aborted = true; break; } } - + curTime = System.currentTimeMillis(); - + // Track requests done if (warmupDone) { requestsDone++; @@ -794,7 +794,7 @@ public void run() { } else { warmupRequests++; } - + // Per-thread periodic progress updates if (curTime > lastUpdate + progressFreq_ms) { if (warmupDone) { @@ -803,13 +803,13 @@ public void run() { lastUpdate = curTime; } else { logger.info(String.format("Requester #%d warming up. " + - " %d warmup requests done. %d/%d seconds of warmup done", + " %d warmup requests done. %d/%d seconds of warmup done", requesterID, warmupRequests, (curTime - warmupStartTime) / 1000, warmupTime)); lastUpdate = curTime; } } - + // Per-thread periodic stat dumps after warmup done if (warmupDone && (lastStatDisplay_ms + displayFreq_ms) <= curTime) { displayStats(lastStatDisplay_ms, curTime); @@ -824,11 +824,11 @@ public void run() { lastStatDisplay_ms = curTime; requestsSinceLastUpdate = 0; logger.info(String.format("Requester #%d warmup finished " + - " after %d warmup requests. 0/%d requests done", + " after %d warmup requests. 0/%d requests done", requesterID, warmupRequests, numRequests)); lastUpdate = curTime; } - + // Enforce time limit if (curTime > endTime) { logger.info(String.format("Requester #%d: time limit of %ds elapsed" + @@ -836,11 +836,11 @@ public void run() { break; } } - + // Do final update of statistics progressTracker.update(requestsSinceLastUpdate); displayStats(lastStatDisplay_ms, System.currentTimeMillis()); - + // Report final stats logger.info("ThreadID = " + requesterID + " total requests = " + requestsDone + @@ -868,7 +868,7 @@ private void displayStats(long lastStatDisplay_ms, long now_ms) { Arrays.asList( LinkBenchOp.MULTIGET_LINK, LinkBenchOp.GET_LINKS_LIST, LinkBenchOp.COUNT_LINK, - LinkBenchOp.UPDATE_LINK, LinkBenchOp.ADD_LINK, + LinkBenchOp.UPDATE_LINK, LinkBenchOp.ADD_LINK, LinkBenchOp.RANGE_SIZE, LinkBenchOp.ADD_NODE, LinkBenchOp.UPDATE_NODE, LinkBenchOp.DELETE_NODE, LinkBenchOp.GET_NODE)); @@ -889,10 +889,10 @@ Link[] getLinkList(long id1, long link_type) throws Exception { if (links != null && links.length >= linkStore.getRangeLimit()) { Link lastLink = links[links.length-1]; if (Level.TRACE.isGreaterOrEqual(debuglevel)) { - logger.trace("Maybe more history for (" + id1 +"," + + logger.trace("Maybe more history for (" + id1 +"," + link_type + " older than " + lastLink.time); } - + addTailCacheEntry(lastLink); } return links; @@ -903,13 +903,13 @@ Link[] getLinkListTail() throws Exception { assert(!listTailHistory.isEmpty()); int choice = rng.nextInt(listTailHistory.size()); Link prevLast = listTailHistory.get(choice); - + // Get links past the oldest last retrieved Link links[] = linkStore.getLinkList(dbid, prevLast.id1, prevLast.link_type, 0, prevLast.time, 1, linkStore.getRangeLimit()); - + if (Level.TRACE.isGreaterOrEqual(debuglevel)) { - logger.trace("getLinkListTail(id1=" + prevLast.id1 + ", link_type=" + logger.trace("getLinkListTail(id1=" + prevLast.id1 + ", link_type=" + prevLast.link_type + ", max_time=" + prevLast.time + " => count=" + (links == null ? 0 : links.length)); } @@ -918,7 +918,7 @@ Link[] getLinkListTail() throws Exception { prevLast.link_type + " older than " + prevLast.time + ": " + (links == null ? 0 : links.length) + " results"); } - + if (links != null && links.length == linkStore.getRangeLimit()) { // There might be yet more history Link last = links[links.length-1]; @@ -930,7 +930,7 @@ Link[] getLinkListTail() throws Exception { listTailHistory.set(choice, last.clone()); } else { // No more history after this, remove from cache - removeTailCacheEntry(choice, null); + removeTailCacheEntry(choice, null); } numHistoryQueries++; return links; @@ -938,7 +938,7 @@ Link[] getLinkListTail() throws Exception { /** * Add a new link to the history cache, unless already present - * @param lastLink the last (i.e. lowest timestamp) link retrieved + * @param lastLink the last (i.e. lowest timestamp) link retrieved */ private void addTailCacheEntry(Link lastLink) { HistoryKey key = new HistoryKey(lastLink); @@ -946,7 +946,7 @@ private void addTailCacheEntry(Link lastLink) { // Already present return; } - + if (listTailHistory.size() < listTailHistoryLimit) { listTailHistory.add(lastLink.clone()); listTailHistoryIndex.put(key, listTailHistory.size() - 1); @@ -974,7 +974,7 @@ private void removeTailCacheEntry(int pos, Link repl) { listTailHistory.set(lastIx, repl); listTailHistoryIndex.put(new HistoryKey(repl), lastIx); } - } else { + } else { if (repl == null) { // Replace with last entry in cache to fill gap repl = listTailHistory.get(listTailHistory.size() - 1); @@ -990,12 +990,12 @@ public static class RequestProgress { static final int THREAD_REPORT_INTERVAL = 250; /** How many ops before a progress update should be printed to console */ private final long interval; - + private final Logger progressLogger; - + private long totalRequests; private final AtomicLong requestsDone; - + private long benchmarkStartTime; private long warmupTime_s; private long timeLimit_s; @@ -1009,11 +1009,11 @@ public RequestProgress(Logger progressLogger, long totalRequests, this.timeLimit_s = timeLimit_s; this.warmupTime_s = warmupTime_s; } - + public void startTimer() { benchmarkStartTime = System.currentTimeMillis() + warmupTime_s * 1000; } - + public long getBenchmarkStartTime() { return benchmarkStartTime; } @@ -1021,7 +1021,7 @@ public long getBenchmarkStartTime() { public void update(long requestIncr) { long curr = requestsDone.addAndGet(requestIncr); long prev = curr - requestIncr; - + if ((curr / interval) > (prev / interval) || curr == totalRequests) { float progressPercent = ((float) curr) / totalRequests * 100; long now = System.currentTimeMillis(); @@ -1034,7 +1034,7 @@ public void update(long requestIncr) { " %.1f/%d secs elapsed: %.1f%% of time limit used", curr, totalRequests, progressPercent, rate, elapsed_s, timeLimit_s, limitPercent)); - + } } } diff --git a/src/java/com/facebook/LinkBench/LinkCount.java b/src/java/com/facebook/LinkBench/LinkCount.java index 4a5b8103..05805029 100644 --- a/src/java/com/facebook/LinkBench/LinkCount.java +++ b/src/java/com/facebook/LinkBench/LinkCount.java @@ -22,7 +22,7 @@ public class LinkCount { public long time; public long version; public long count; - public LinkCount(long id1, long link_type, + public LinkCount(long id1, long link_type, long time, long version, long init_count) { super(); this.id1 = id1; diff --git a/src/java/com/facebook/LinkBench/LinkStore.java b/src/java/com/facebook/LinkBench/LinkStore.java index 0530874b..d8623667 100644 --- a/src/java/com/facebook/LinkBench/LinkStore.java +++ b/src/java/com/facebook/LinkBench/LinkStore.java @@ -29,10 +29,10 @@ public abstract class LinkStore { // visibility public static final byte VISIBILITY_HIDDEN = 0; public static final byte VISIBILITY_DEFAULT = 1; - + public static final int MAX_OPTYPES = LinkBenchOp.values().length; public static final int DEFAULT_LIMIT = 10000; - + public static final long MAX_LINK_DATA = 255; /** Controls the current setting for range limit */ @@ -41,8 +41,8 @@ public abstract class LinkStore { /** The default constructor */ public LinkStore() { this.rangeLimit = DEFAULT_LIMIT; - } - + } + public int getRangeLimit() { return rangeLimit; } @@ -61,7 +61,7 @@ public abstract void initialize(Properties p, public abstract void close(); // this is invoked when an error happens in case connection needs to be - // cleaned up, reset, reopened, whatever + // cleaned up, reset, reopened, whatever public abstract void clearErrors(int threadID); /** @@ -74,7 +74,7 @@ public abstract void initialize(Properties p, * @throws Exception */ public abstract boolean addLink(String dbid, Link a, boolean noinverse) throws Exception; - + /** * Delete link identified by parameters from store * @param dbid @@ -87,9 +87,9 @@ public abstract void initialize(Properties p, * purposes only. * @throws Exception */ - public abstract boolean deleteLink(String dbid, long id1, long link_type, long id2, - boolean noinverse, boolean expunge) throws Exception; - + public abstract boolean deleteLink(String dbid, long id1, long link_type, + long id2, boolean noinverse, boolean expunge) throws Exception; + /** * Update a link in the database, or add if not found * @param dbid @@ -99,7 +99,7 @@ public abstract boolean deleteLink(String dbid, long id1, long link_type, long i * optional, for informational purposes only. * @throws Exception */ - public abstract boolean updateLink(String dbid, Link a, boolean noinverse) + public abstract boolean updateLink(String dbid, Link a, boolean noinverse) throws Exception; /** @@ -112,16 +112,16 @@ public abstract boolean updateLink(String dbid, Link a, boolean noinverse) * @return * @throws Exception */ - public abstract Link getLink(String dbid, long id1, long link_type, long id2) + public abstract Link getLink(String dbid, long id1, long link_type, long id2) throws Exception; - + /** * Lookup multiple links: same as getlink but retrieve - * multiple ids + * multiple ids * @return list of matching links found, in any order */ public Link[] multigetLinks(String dbid, long id1, long link_type, - long id2s[]) + long id2s[]) throws Exception { // Default implementation ArrayList res = new ArrayList(id2s.length); @@ -144,7 +144,7 @@ public Link[] multigetLinks(String dbid, long id1, long link_type, * if no matching links * @throws Exception */ - public abstract Link[] getLinkList(String dbid, long id1, long link_type) + public abstract Link[] getLinkList(String dbid, long id1, long link_type) throws Exception; @@ -162,29 +162,29 @@ public abstract Link[] getLinkList(String dbid, long id1, long link_type) * if no matching links * @throws Exception */ - public abstract Link[] getLinkList(String dbid, long id1, long link_type, - long minTimestamp, long maxTimestamp, + public abstract Link[] getLinkList(String dbid, long id1, long link_type, + long minTimestamp, long maxTimestamp, int offset, int limit) throws Exception; // count the #links public abstract long countLinks(String dbid, long id1, long link_type) throws Exception; - - /** + + /** * @return 0 if it doesn't support addBulkLinks and recalculateCounts methods - * If it does support them, return the maximum number of links that + * If it does support them, return the maximum number of links that * can be added at a time */ public int bulkLoadBatchSize() { return 0; } - + /** Add a batch of links without updating counts */ public void addBulkLinks(String dbid, List a, boolean noinverse) throws Exception { throw new UnsupportedOperationException("addBulkLinks not supported for " + - "LinkStore subclass " + this.getClass().getName()); + "LinkStore subclass " + this.getClass().getName()); } - + /** Add a batch of counts */ public void addBulkCounts(String dbid, List a) throws Exception { diff --git a/src/java/com/facebook/LinkBench/LinkStoreMysql.java b/src/java/com/facebook/LinkBench/LinkStoreMysql.java index 297311f3..ef4df32e 100644 --- a/src/java/com/facebook/LinkBench/LinkStoreMysql.java +++ b/src/java/com/facebook/LinkBench/LinkStoreMysql.java @@ -34,7 +34,7 @@ import org.apache.log4j.Logger; public class LinkStoreMysql extends GraphStore { - + /* MySql database server configuration keys */ public static final String CONFIG_HOST = "host"; public static final String CONFIG_PORT = "port"; @@ -42,15 +42,15 @@ public class LinkStoreMysql extends GraphStore { public static final String CONFIG_PASSWORD = "password"; public static final String CONFIG_BULK_INSERT_BATCH = "mysql_bulk_insert_batch"; public static final String CONFIG_DISABLE_BINLOG_LOAD = "mysql_disable_binlog_load"; - + public static final int DEFAULT_BULKINSERT_SIZE = 1024; - + private static final boolean INTERNAL_TESTING = false; - + String linktable; String counttable; String nodetable; - + String host; String user; String pwd; @@ -60,7 +60,7 @@ public class LinkStoreMysql extends GraphStore { Level debuglevel; Connection conn; Statement stmt; - + private Phase phase; int bulkInsertSize = DEFAULT_BULKINSERT_SIZE; @@ -87,7 +87,7 @@ public void initialize(Properties props, Phase currentPhase, logger.error(msg); throw new RuntimeException(msg); } - + nodetable = props.getProperty(Config.NODE_TABLE); if (nodetable.equals("")) { // For now, don't assume that nodetable is provided @@ -96,13 +96,13 @@ public void initialize(Properties props, Phase currentPhase, logger.error(msg); throw new RuntimeException(msg); } - + host = ConfigUtil.getPropertyRequired(props, CONFIG_HOST); user = ConfigUtil.getPropertyRequired(props, CONFIG_USER); pwd = ConfigUtil.getPropertyRequired(props, CONFIG_PASSWORD); port = props.getProperty(CONFIG_PORT); defaultDB = ConfigUtil.getPropertyRequired(props, Config.DBID); - + if (port == null || port.equals("")) port = "3306"; //use default port debuglevel = ConfigUtil.getDebugLevel(props); phase = currentPhase; @@ -114,7 +114,7 @@ public void initialize(Properties props, Phase currentPhase, disableBinLogForLoad = ConfigUtil.getBool(props, CONFIG_DISABLE_BINLOG_LOAD); } - + // connect try { openConnection(); @@ -130,7 +130,7 @@ public void initialize(Properties props, Phase currentPhase, private void openConnection() throws Exception { conn = null; stmt = null; - + String jdbcUrl = "jdbc:mysql://"+ host + ":" + port + "/"; if (defaultDB != null) { jdbcUrl += defaultDB; @@ -151,13 +151,13 @@ private void openConnection() throws Exception { conn.setAutoCommit(false); stmt = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); - + if (phase == Phase.LOAD && disableBinLogForLoad) { // Turn binary logging off for duration of connection stmt.executeUpdate("SET SESSION sql_log_bin=0"); } } - + @Override public void close() { try { @@ -182,7 +182,7 @@ public void clearErrors(int threadID) { return; } } - + /** * Set of all JDBC SQLState strings that indicate a transient MySQL error * that should be handled by retrying @@ -200,7 +200,7 @@ private static HashSet populateRetrySQLStates() { states.add("40001"); // ER_LOCK_DEADLOCK return states; } - + /** * Handle SQL exception by logging error and selecting how to respond * @param ex SQLException thrown by MySQL JDBC driver @@ -212,7 +212,7 @@ private boolean processSQLException(SQLException ex, String op) { "operation: " + op + ". "; msg += "Message was: '" + ex.getMessage() + "'. "; msg += "SQLState was: " + ex.getSQLState() + ". "; - + if (retry) { msg += "Error is probably transient, retrying operation."; logger.warn(msg); @@ -222,7 +222,7 @@ private boolean processSQLException(SQLException ex, String op) { } return retry; } - + // get count for testing purpose private void testCount(Statement stmt, String dbid, String assoctable, String counttable, @@ -269,7 +269,7 @@ public boolean addLink(String dbid, Link l, boolean noinverse) } } } - + private boolean addLinkImpl(String dbid, Link l, boolean noinverse) throws Exception { @@ -284,7 +284,7 @@ private boolean addLinkImpl(String dbid, Link l, boolean noinverse) // if the link is already there then update its visibility // only update visibility; skip updating time, version, etc. - + int nrows = addLinksNoCount(dbid, Collections.singletonList(l)); if (Level.TRACE.isGreaterOrEqual(debuglevel)) { @@ -314,7 +314,7 @@ private boolean addLinkImpl(String dbid, Link l, boolean noinverse) break; case 2: - // a visibility was changed from VISIBILITY_HIDDEN to DEFAULT + // a visibility was changed from VISIBILITY_HIDDEN to DEFAULT // or vice-versa // --> need to update both counttable and other data if (l.visibility == VISIBILITY_DEFAULT) { @@ -405,10 +405,10 @@ private int addLinksNoCount(String dbid, List links) } else { sb.append(','); } - sb.append("(" + l.id1 + - ", " + l.id2 + - ", " + l.link_type + - ", " + l.visibility + + sb.append("(" + l.id1 + + ", " + l.id2 + + ", " + l.link_type + + ", " + l.visibility + ", " + stringLiteral(l.data) + ", " + l.time + ", " + l.version + ")"); @@ -422,7 +422,7 @@ private int addLinksNoCount(String dbid, List links) int nrows = stmt.executeUpdate(insert); return nrows; } - + @Override public boolean deleteLink(String dbid, long id1, long link_type, long id2, boolean noinverse, boolean expunge) @@ -547,7 +547,7 @@ else if (visibility == VISIBILITY_DEFAULT) { @Override public boolean updateLink(String dbid, Link l, boolean noinverse) throws Exception { - // Retry logic is in addLink + // Retry logic is in addLink boolean added = addLink(dbid, l, noinverse); return !added; // return true if updated instead of added } @@ -567,7 +567,7 @@ public Link getLink(String dbid, long id1, long link_type, long id2) } } } - + private Link getLinkImpl(String dbid, long id1, long link_type, long id2) throws Exception { Link res[] = multigetLinks(dbid, id1, link_type, new long[] {id2}); @@ -578,7 +578,7 @@ private Link getLinkImpl(String dbid, long id1, long link_type, long id2) @Override - public Link[] multigetLinks(String dbid, long id1, long link_type, + public Link[] multigetLinks(String dbid, long id1, long link_type, long[] id2s) throws Exception { while (true) { try { @@ -590,8 +590,8 @@ public Link[] multigetLinks(String dbid, long id1, long link_type, } } } - - private Link[] multigetLinksImpl(String dbid, long id1, long link_type, + + private Link[] multigetLinksImpl(String dbid, long id1, long link_type, long[] id2s) throws Exception { StringBuilder querySB = new StringBuilder(); querySB.append(" select id1, id2, link_type," + @@ -608,22 +608,22 @@ private Link[] multigetLinksImpl(String dbid, long id1, long link_type, } querySB.append(id2); } - + querySB.append("); commit;"); String query = querySB.toString(); if (Level.TRACE.isGreaterOrEqual(debuglevel)) { logger.trace("Query is " + query); } - + ResultSet rs = stmt.executeQuery(query); - + // Get the row count to allocate result array assert(rs.getType() != ResultSet.TYPE_FORWARD_ONLY); rs.last(); int count = rs.getRow(); rs.beforeFirst(); - + Link results[] = new Link[count]; int i = 0; while (rs.next()) { @@ -661,13 +661,13 @@ public Link[] getLinkList(String dbid, long id1, long link_type, } } } - + private Link[] getLinkListImpl(String dbid, long id1, long link_type, long minTimestamp, long maxTimestamp, int offset, int limit) throws Exception { String query = " select id1, id2, link_type," + - " visibility, data, time," + + " visibility, data, time," + " version from " + dbid + "." + linktable + " where id1 = " + id1 + " and link_type = " + link_type + " and time >= " + minTimestamp + @@ -681,7 +681,7 @@ private Link[] getLinkListImpl(String dbid, long id1, long link_type, } ResultSet rs = stmt.executeQuery(query); - + // Find result set size // be sure we fast forward to find result set size assert(rs.getType() != ResultSet.TYPE_FORWARD_ONLY); @@ -696,7 +696,7 @@ private Link[] getLinkListImpl(String dbid, long id1, long link_type, if (count == 0) { return null; } - + // Fetch the link data Link links[] = new Link[count]; int i = 0; @@ -735,7 +735,7 @@ public long countLinks(String dbid, long id1, long link_type) } } } - + private long countLinksImpl(String dbid, long id1, long link_type) throws Exception { long count = 0; @@ -782,13 +782,13 @@ public void addBulkLinks(String dbid, List links, boolean noinverse) } } } - + private void addBulkLinksImpl(String dbid, List links, boolean noinverse) throws Exception { if (Level.TRACE.isGreaterOrEqual(debuglevel)) { logger.trace("addBulkLinks: " + links.size() + " links"); } - + addLinksNoCount(dbid, links); conn.commit(); } @@ -807,7 +807,7 @@ public void addBulkCounts(String dbid, List counts) } } } - + private void addBulkCountsImpl(String dbid, List counts) throws Exception { if (Level.TRACE.isGreaterOrEqual(debuglevel)) { @@ -815,8 +815,8 @@ private void addBulkCountsImpl(String dbid, List counts) } if (counts.size() == 0) return; - - StringBuilder sqlSB = new StringBuilder(); + + StringBuilder sqlSB = new StringBuilder(); sqlSB.append("REPLACE INTO " + dbid + "." + counttable + "(id, link_type, count, time, version) " + "VALUES "); @@ -833,7 +833,7 @@ private void addBulkCountsImpl(String dbid, List counts) ", " + count.time + ", " + count.version + ")"); } - + String sql = sqlSB.toString(); if (Level.TRACE.isGreaterOrEqual(debuglevel)) { logger.trace(sql); @@ -841,11 +841,11 @@ private void addBulkCountsImpl(String dbid, List counts) stmt.executeUpdate(sql); conn.commit(); } - + private void checkNodeTableConfigured() throws Exception { if (this.nodetable == null) { throw new Exception("Nodetable not specified: cannot perform node" + - " operation"); + " operation"); } } @@ -856,7 +856,7 @@ public void resetNodeStore(String dbid, long startID) throws Exception { stmt.execute(String.format("TRUNCATE TABLE `%s`.`%s`;", dbid, nodetable)); stmt.execute(String.format("ALTER TABLE `%s`.`%s` " + - "AUTO_INCREMENT = %d;", dbid, nodetable, startID)); + "AUTO_INCREMENT = %d;", dbid, nodetable, startID)); } @Override @@ -871,7 +871,7 @@ public long addNode(String dbid, Node node) throws Exception { } } } - + private long addNodeImpl(String dbid, Node node) throws Exception { long ids[] = bulkAddNodes(dbid, Collections.singletonList(node)); assert(ids.length == 1); @@ -890,7 +890,7 @@ public long[] bulkAddNodes(String dbid, List nodes) throws Exception { } } } - + private long[] bulkAddNodesImpl(String dbid, List nodes) throws Exception { checkNodeTableConfigured(); StringBuilder sql = new StringBuilder(); @@ -904,7 +904,7 @@ private long[] bulkAddNodesImpl(String dbid, List nodes) throws Exception } else { sql.append(","); } - sql.append("(" + node.type + "," + node.version + + sql.append("(" + node.type + "," + node.version + "," + node.time + "," + stringLiteral(node.data) + ")"); } sql.append("; commit;"); @@ -913,22 +913,22 @@ private long[] bulkAddNodesImpl(String dbid, List nodes) throws Exception } stmt.executeUpdate(sql.toString(), Statement.RETURN_GENERATED_KEYS); ResultSet rs = stmt.getGeneratedKeys(); - + long newIds[] = new long[nodes.size()]; // Find the generated id int i = 0; while (rs.next() && i < nodes.size()) { newIds[i++] = rs.getLong(1); } - + if (i != nodes.size()) { throw new Exception("Wrong number of generated keys on insert: " + " expected " + nodes.size() + " actual " + i); } - + assert(!rs.next()); // check done rs.close(); - + return newIds; } @@ -944,7 +944,7 @@ public Node getNode(String dbid, int type, long id) throws Exception { } } } - + private Node getNodeImpl(String dbid, int type, long id) throws Exception { checkNodeTableConfigured(); ResultSet rs = stmt.executeQuery( @@ -979,20 +979,20 @@ public boolean updateNode(String dbid, Node node) throws Exception { } } } - + private boolean updateNodeImpl(String dbid, Node node) throws Exception { checkNodeTableConfigured(); String sql = "UPDATE `" + dbid + "`.`" + nodetable + "`" + " SET " + "version=" + node.version + ", time=" + node.time - + ", data=" + stringLiteral(node.data) + + + ", data=" + stringLiteral(node.data) + " WHERE id=" + node.id + " AND type=" + node.type + "; commit;"; - + if (Level.TRACE.isGreaterOrEqual(debuglevel)) { logger.trace(sql); - } - + } + int rows = stmt.executeUpdate(sql); - + if (rows == 1) return true; else if (rows == 0) return false; else throw new Exception("Did not expect " + rows + "affected rows: only " @@ -1011,27 +1011,27 @@ public boolean deleteNode(String dbid, int type, long id) throws Exception { } } } - + private boolean deleteNodeImpl(String dbid, int type, long id) throws Exception { checkNodeTableConfigured(); int rows = stmt.executeUpdate( "DELETE FROM `" + dbid + "`.`" + nodetable + "` " + "WHERE id=" + id + " and type =" + type + "; commit;"); - + if (rows == 0) { return false; } else if (rows == 1) { return true; } else { throw new Exception(rows + " rows modified on delete: should delete " + - "at most one"); + "at most one"); } } - - /** + + /** * Convert a byte array into a valid mysql string literal, assuming that * it will be inserted into a column with latin-1 encoding. - * Based on information at + * Based on information at * http://dev.mysql.com/doc/refman/5.1/en/string-literals.html * @param arr * @return @@ -1079,7 +1079,7 @@ private static String stringLiteral(byte arr[]) { /** * Create a mysql hex string literal from array: - * E.g. [0xf, bc, 4c, 4] converts to x'0fbc4c03' + * E.g. [0xf, bc, 4c, 4] converts to x'0fbc4c03' * @param arr * @return the mysql hex literal including quotes */ diff --git a/src/java/com/facebook/LinkBench/LinkStoreRocksDb.java b/src/java/com/facebook/LinkBench/LinkStoreRocksDb.java index a6318a37..d9b3b7ac 100644 --- a/src/java/com/facebook/LinkBench/LinkStoreRocksDb.java +++ b/src/java/com/facebook/LinkBench/LinkStoreRocksDb.java @@ -49,9 +49,9 @@ * This file implements Linkbench methods for loading/requesting data to rocksDb * database server by calling thrift apis after creating 2 java thrift clients * through swift : assocClient for the link operations and nodeClient for the - * node operations. assocClient interacts on port 'port' on the database = + * node operations. assocClient interacts on port 'port' on the database = * dbid + "assocs" AND nodeClient interacts on port 'port'+1 on the database= - * dbid + "nodes" + * dbid + "nodes" */ public class LinkStoreRocksDb extends GraphStore { @@ -64,10 +64,10 @@ public class LinkStoreRocksDb extends GraphStore { public static final String CONFIG_PORT = "port"; public static final String CONFIG_USER = "user"; public static final String CONFIG_PASSWORD = "password"; - + public static final int DEFAULT_BULKINSERT_SIZE = 1024; private static final boolean INTERNAL_TESTING = false; - + private static int totalThreads = 0; String host; @@ -76,12 +76,12 @@ public class LinkStoreRocksDb extends GraphStore { String port; Level debuglevel; - + int bulkInsertSize = DEFAULT_BULKINSERT_SIZE; private final Logger logger = Logger.getLogger(ConfigUtil.LINKBENCH_LOGGER); - private void openConnection() throws Exception { + private void openConnection() throws Exception { try { assocClient = clientManager.createClient( new FramedClientConnector(fromParts(host, Integer.parseInt(port))), @@ -121,7 +121,7 @@ public void close() { if (clientManager != null) clientManager.close(); } catch (IOException ioex) { - logger.error("Error while closing client connection: " + ioex); + logger.error("Error while closing client connection: " + ioex); } } @@ -143,18 +143,18 @@ public LinkStoreRocksDb(Properties props) throws IOException, Exception { super(); initialize(props, Phase.LOAD, 0); } - + public void clearErrors(int threadID) { logger.warn("Reopening Rocksdb connection in threadID " + threadID); - try { - close(); - openConnection(); - } catch (Throwable e) { + try { + close(); + openConnection(); + } catch (Throwable e) { logger.error("Error in Reopen!" + e); - e.printStackTrace(); + e.printStackTrace(); } } - + @Override public boolean addLink(String dbid, Link l, boolean noinverse) { try { @@ -164,7 +164,7 @@ public boolean addLink(String dbid, Link l, boolean noinverse) { return false; } } - + private boolean addLinkImpl(String dbid, Link l, boolean noinverse) throws Exception { @@ -200,7 +200,7 @@ private boolean addLinksNoCount(String dbid, List links) } return true; } - + @Override public boolean deleteLink(String dbid, long id1, long link_type, long id2, boolean noinverse, boolean expunge) { @@ -228,7 +228,7 @@ private boolean deleteLinkImpl(String dbid, long id1, long link_type, @Override public boolean updateLink(String dbid, Link l, boolean noinverse) throws Exception { - // Retry logic is in addLink + // Retry logic is in addLink boolean added = addLink(dbid, l, noinverse); return !added; // return true if updated instead of added } @@ -244,7 +244,7 @@ public Link getLink(String dbid, long id1, long link_type, long id2) { return null; } } - + private Link getLinkImpl(String dbid, long id1, long link_type, long id2) throws Exception { Link res[] = multigetLinks(dbid, id1, link_type, new long[] {id2}); @@ -256,7 +256,7 @@ private Link getLinkImpl(String dbid, long id1, long link_type, long id2) @Override - public Link[] multigetLinks(String dbid, long id1, long link_type, + public Link[] multigetLinks(String dbid, long id1, long link_type, long[] id2s) { try { return multigetLinksImpl(dbid, id1, link_type, id2s); @@ -265,9 +265,9 @@ public Link[] multigetLinks(String dbid, long id1, long link_type, return null; } } - - private Link[] multigetLinksImpl(String dbid, long id1, long link_type, - long[] id2s) throws Exception { + + private Link[] multigetLinksImpl(String dbid, long id1, long link_type, + long[] id2s) throws Exception { List l = new ArrayList(); for (int i = 0; i < id2s.length; i++) { l.add(new Long(id2s[i])); @@ -304,14 +304,14 @@ public Link[] getLinkList(String dbid, long id1, long link_type, return null; } } - + private Link[] getLinkListImpl(String dbid, long id1, long link_type, long minTimestamp, long maxTimestamp, int offset, int limit) throws Exception { dbid += "assocs"; List tr = assocClient.TaoAssocRangeGet( dbid.getBytes(), link_type, id1, maxTimestamp, minTimestamp, - Long.valueOf(offset), Long.valueOf(limit)); + Long.valueOf(offset), Long.valueOf(limit)); Link results[] = new Link[tr.size()]; int i = 0; for (TaoAssocGetResult tar : tr) { @@ -333,7 +333,7 @@ public long countLinks(String dbid, long id1, long link_type) { return -1; } } - + private long countLinksImpl(String dbid, long id1, long link_type) throws Exception { long count = 0; @@ -363,7 +363,7 @@ public void addBulkLinks(String dbid, List links, boolean noinverse) { logger.error("addBulkLinks failed! " + ex); } } - + private void addBulkLinksImpl(String dbid, List links, boolean noinverse) throws Exception { if (Level.TRACE.isGreaterOrEqual(debuglevel)) { @@ -380,7 +380,7 @@ public void addBulkCounts(String dbid, List counts) { logger.error("addbulkCounts failed! " + ex); } } - + private void addBulkCountsImpl(String dbid, List counts) throws Exception { if (Level.TRACE.isGreaterOrEqual(debuglevel)) { @@ -388,7 +388,7 @@ private void addBulkCountsImpl(String dbid, List counts) } if (counts.size() == 0) return; - + WriteOptions wopts = new WriteOptions(); wopts.setSync(false); List batchCounts = new ArrayList(); @@ -402,17 +402,17 @@ private void addBulkCountsImpl(String dbid, List counts) System.arraycopy(linkType, 0, ckey, id1.length, linkType.length); char c = 'c'; ckey[ckey.length - 1] = (byte)c; - + byte[] countValue = ByteBuffer.allocate(8).putLong( count.count).array(); - + Kv keyvalue = new Kv(); keyvalue.setKey(ckey); keyvalue.setValue(countValue); batchCounts.add(keyvalue); - } + } dbid += "assocs"; - assocClient.Write(dbid.getBytes(), batchCounts, wopts); + assocClient.Write(dbid.getBytes(), batchCounts, wopts); } @Override @@ -429,7 +429,7 @@ public long addNode(String dbid, Node node) { return -1; } } - + private long addNodeImpl(String dbid, Node node) throws Exception { long ids[] = bulkAddNodes(dbid, Collections.singletonList(node)); assert(ids.length == 1); @@ -445,7 +445,7 @@ public long[] bulkAddNodes(String dbid, List nodes) throws Exception { return null; } } - + private long[] bulkAddNodesImpl(String dbid, List nodes) throws Exception { long newIds[] = new long[nodes.size()]; @@ -479,11 +479,11 @@ public Node getNode(String dbid, int type, long id) { return null; } } - + private Node getNodeImpl(String dbid, int type, long id) throws Exception { ReadOptions ropts = new ReadOptions(); dbid += "nodes"; - RocksGetResponse rgr = + RocksGetResponse rgr = nodeClient.Get( dbid.getBytes(), ByteBuffer.allocate(8).putLong(id).array(), ropts); if (rgr.getRetCode().getState() == (Code.K_NOT_FOUND)) { @@ -515,7 +515,7 @@ public boolean updateNode(String dbid, Node node) throws Exception { return false; } } - + private boolean updateNodeImpl(String dbid, Node node) throws Exception { return addNode(dbid, node) == 1; } @@ -529,7 +529,7 @@ public boolean deleteNode(String dbid, int type, long id) throws Exception { return false; } } - + private boolean deleteNodeImpl(String dbid, int type, long id) throws Exception { WriteOptions wopts = new WriteOptions(); diff --git a/src/java/com/facebook/LinkBench/MemoryLinkStore.java b/src/java/com/facebook/LinkBench/MemoryLinkStore.java index d9e6c4b0..3fa15d51 100644 --- a/src/java/com/facebook/LinkBench/MemoryLinkStore.java +++ b/src/java/com/facebook/LinkBench/MemoryLinkStore.java @@ -27,7 +27,7 @@ /** * Simple in-memory implementation of GraphStore * Not efficient or optimized at all, just for testing purposes. - * + * * MemoryLinkStore instances sharing the same data can be created * using the newInstance() method. * MemoryLinkStore can be accessed concurrently from multiple threads, @@ -38,7 +38,7 @@ public class MemoryLinkStore extends GraphStore { private static class LinkLookupKey { final long id1; final long link_type; - + public LinkLookupKey(long id1, long link_type) { super(); this.id1 = id1; @@ -50,7 +50,7 @@ public boolean equals(Object other) { if (!(other instanceof LinkLookupKey)) { return false; } - LinkLookupKey other2 = (LinkLookupKey)other; + LinkLookupKey other2 = (LinkLookupKey)other; return id1 == other2.id1 && link_type == other2.link_type; } @@ -59,7 +59,7 @@ public int hashCode() { return Long.valueOf(id1).hashCode() ^ Long.valueOf(link_type).hashCode(); } } - + /** Order links from most to least recent */ private static class LinkTimeStampComparator implements Comparator { @@ -68,7 +68,7 @@ public int compare(Link l1, Link l2) { // ascending order of id1 if (l1.id1 != l2.id1) { if (l1.id1 < l2.id2) { - return -1; + return -1; } else { return 1; } @@ -76,12 +76,12 @@ public int compare(Link l1, Link l2) { if (l1.time != l2.time) { // descending order of time if (l1.time < l2.time) { - return 1; + return 1; } else { return -1; } } - + // ascending order of id2 if (l1.id2 == l2.id2) { return 0; @@ -92,66 +92,66 @@ public int compare(Link l1, Link l2) { } } } - + /** * Class for allocating IDs and storing objects */ private static class NodeDB { private long nextID; // Next id to allocate Map data = new HashMap(); - + /** Construct a new instance allocating ids from 1 */ NodeDB() { this(1); } - + NodeDB(long startID) { this.nextID = startID; } - + long allocateID() { return nextID++; } - + void reset(long startID) { nextID = startID; data.clear(); } } - - + + /** Simple implementation of LinkStore with nested maps and a set of - * links sorted by timestamp: + * links sorted by timestamp: * dbid -> (id1, assoc_type) -> links */ private final Map>> linkdbs; - + private final Map nodedbs; - + /** * Storage for objects */ private static final Comparator LINK_COMPARATOR = new LinkTimeStampComparator(); - - /** - * Create a new MemoryLinkStore instance with fresh data + + /** + * Create a new MemoryLinkStore instance with fresh data */ public MemoryLinkStore() { this(new HashMap>>(), new HashMap()); } - - /** - * Create a new MemoryLinkStore handle sharing data with existing instance + + /** + * Create a new MemoryLinkStore handle sharing data with existing instance */ private MemoryLinkStore(Map>> linkdbs, Map nodedbs) { this.linkdbs = linkdbs; this.nodedbs = nodedbs; } - + /** - * Find a list of links based on + * Find a list of links based on * @param dbid * @param id1 * @param link_type @@ -195,7 +195,7 @@ private TreeSet newSortedLinkSet() { public MemoryLinkStore newHandle() { return new MemoryLinkStore(linkdbs, nodedbs); } - + @Override public void initialize(Properties p, Phase currentPhase, int threadId) throws IOException, Exception { @@ -213,7 +213,7 @@ public void clearErrors(int threadID) { public boolean addLink(String dbid, Link a, boolean noinverse) throws Exception { synchronized (linkdbs) { SortedSet links = findLinkByKey(dbid, a.id1, a.link_type, true); - + boolean exists = false; // Check for duplicates Iterator it = links.iterator(); @@ -226,7 +226,7 @@ public boolean addLink(String dbid, Link a, boolean noinverse) throws Exception } // Clone argument before inserting links.add(a.clone()); - + /*System.err.println(String.format("added link (%d, %d, %d), %d in list", a.id1, a.link_type, a.id2, links.size()));*/ return !exists; @@ -273,7 +273,7 @@ public boolean updateLink(String dbid, Link a, boolean noinverse) } } } - + // Throw error if updating non-existing link throw new Exception(String.format("Link not found: (%d, %d, %d)", a.id1, a.link_type, a.id2)); @@ -315,7 +315,7 @@ public Link[] getLinkList(String dbid, long id1, long link_type, // Do a first pass to find size of result array int matching = 0; for (Link l: linkSet) { - if (l.visibility == VISIBILITY_DEFAULT && + if (l.visibility == VISIBILITY_DEFAULT && l.time >= minTimestamp && l.time <= maxTimestamp) { if (skipped < offset) { skipped++; @@ -331,19 +331,19 @@ public Link[] getLinkList(String dbid, long id1, long link_type, return null; } Link res[] = new Link[matching]; - + // Iterate in desc order of timestamp, break ties by id2 int i = 0; skipped = 0; for (Link l: linkSet) { - if (l.visibility == VISIBILITY_DEFAULT && + if (l.visibility == VISIBILITY_DEFAULT && l.time >= minTimestamp && l.time <= maxTimestamp) { if (skipped < offset) { skipped++; continue; } res[i++] = l; - + if (i >= limit) { break; } @@ -369,7 +369,7 @@ public long countLinks(String dbid, long id1, long link_type) visible++; } } - + /*System.err.println( String.format("Lookup (%d, %d): %d visible, %d total", id1, link_type, visible, linkSet.size()));*/ diff --git a/src/java/com/facebook/LinkBench/Node.java b/src/java/com/facebook/LinkBench/Node.java index b8c343c5..2b260be1 100644 --- a/src/java/com/facebook/LinkBench/Node.java +++ b/src/java/com/facebook/LinkBench/Node.java @@ -27,7 +27,7 @@ public class Node { public long version; public int time; public byte data[]; - + public Node(long id, int type, long version, int time, byte data[]) { super(); @@ -50,7 +50,7 @@ public boolean equals(Object other) { return id == o.id && type == o.type && version == o.version && time == o.time && Arrays.equals(data, o.data); } - + public String toString() { return "Node(" + "id=" + id + ",type=" + type + ",version=" + version + "," + "timestamp=" + time + ",data=" diff --git a/src/java/com/facebook/LinkBench/NodeLoader.java b/src/java/com/facebook/LinkBench/NodeLoader.java index 2fc743f4..6b481b0d 100644 --- a/src/java/com/facebook/LinkBench/NodeLoader.java +++ b/src/java/com/facebook/LinkBench/NodeLoader.java @@ -32,7 +32,7 @@ /** * Load class for generating node data - * + * * This is separate from link loading because we can't have multiple parallel * loaders loading nodes, as the order of IDs being assigned would be messed up * @author tarmstrong @@ -48,28 +48,28 @@ public class NodeLoader implements Runnable { // Data generation settings private final DataGenerator nodeDataGen; private final LogNormalDistribution nodeDataLength; - + private final Level debuglevel; private final int loaderId; private final SampledStats stats; private final LatencyStats latencyStats; - + private long startTime_ms; - + private long nodesLoaded = 0; private long totalNodes = 0; - + /** Next node count to report on */ private long nextReport = 0; - + /** Last time stat update displayed */ private long lastDisplayTime_ms; /** How often to display stat updates */ private final long displayFreq_ms; - - + + public NodeLoader(Properties props, Logger logger, NodeStore nodeStore, Random rng, LatencyStats latencyStats, PrintStream csvStreamOut, int loaderId) { @@ -80,10 +80,10 @@ public NodeLoader(Properties props, Logger logger, this.rng = rng; this.latencyStats = latencyStats; this.loaderId = loaderId; - + double medianDataLength = ConfigUtil.getDouble(props, Config.NODE_DATASIZE); nodeDataLength = new LogNormalDistribution(); - nodeDataLength.init(0, NodeStore.MAX_NODE_DATA, medianDataLength, + nodeDataLength.init(0, NodeStore.MAX_NODE_DATA, medianDataLength, Config.NODE_DATASIZE_SIGMA); try { @@ -93,13 +93,13 @@ public NodeLoader(Properties props, Logger logger, nodeDataGen.init(props, Config.NODE_ADD_DATAGEN_PREFIX); } catch (ClassNotFoundException ex) { logger.error(ex); - throw new LinkBenchConfigError("Error loading data generator class: " + throw new LinkBenchConfigError("Error loading data generator class: " + ex.getMessage()); } - + debuglevel = ConfigUtil.getDebugLevel(props); dbid = ConfigUtil.getPropertyRequired(props, Config.DBID); - + displayFreq_ms = ConfigUtil.getLong(props, Config.DISPLAY_FREQ) * 1000; int maxsamples = ConfigUtil.getInt(props, Config.MAX_STAT_SAMPLES); @@ -109,7 +109,7 @@ public NodeLoader(Properties props, Logger logger, @Override public void run() { logger.info("Starting loader thread #" + loaderId + " loading nodes"); - + try { this.nodeStore.initialize(props, Phase.LOAD, loaderId); } catch (Exception e) { @@ -122,10 +122,10 @@ public void run() { nodeStore.resetNodeStore(dbid, ConfigUtil.getLong(props, Config.MIN_ID)); } catch (Exception e) { logger.error("Error while resetting IDs, cannot proceed with " + - "node loading", e); + "node loading", e); return; } - + int bulkLoadBatchSize = nodeStore.bulkLoadBatchSize(); ArrayList nodeLoadBuffer = new ArrayList(bulkLoadBatchSize); @@ -137,7 +137,7 @@ public void run() { lastDisplayTime_ms = startTime_ms; for (long id = startId; id < maxId; id++) { genNode(rng, id, nodeLoadBuffer, bulkLoadBatchSize); - + long now = System.currentTimeMillis(); if (lastDisplayTime_ms + displayFreq_ms <= now) { displayAndResetStats(); @@ -145,7 +145,7 @@ public void run() { } // Load any remaining data loadNodes(nodeLoadBuffer); - + logger.info("Loading of nodes [" + startId + "," + maxId + ") done"); displayAndResetStats(); nodeStore.close(); @@ -153,7 +153,7 @@ public void run() { private void displayAndResetStats() { long now = System.currentTimeMillis(); - stats.displayStats(lastDisplayTime_ms, now, + stats.displayStats(lastDisplayTime_ms, now, Arrays.asList(LinkBenchOp.LOAD_NODE_BULK)); stats.resetSamples(); lastDisplayTime_ms = now; @@ -166,7 +166,7 @@ private void displayAndResetStats() { */ private void genNode(Random rng, long id1, ArrayList nodeLoadBuffer, int bulkLoadBatchSize) { - int dataLength = (int)nodeDataLength.choose(rng); + int dataLength = (int)nodeDataLength.choose(rng); Node node = new Node(id1, LinkStore.DEFAULT_NODE_TYPE, System.currentTimeMillis(), 1, nodeDataGen.fill(rng, new byte[dataLength])); nodeLoadBuffer.add(node); @@ -175,7 +175,7 @@ private void genNode(Random rng, long id1, ArrayList nodeLoadBuffer, nodeLoadBuffer.clear(); } } - + private void loadNodes(ArrayList nodeLoadBuffer) { long actualIds[] = null; long timestart = System.nanoTime(); @@ -183,29 +183,29 @@ private void loadNodes(ArrayList nodeLoadBuffer) { actualIds = nodeStore.bulkAddNodes(dbid, nodeLoadBuffer); long timetaken = (System.nanoTime() - timestart); nodesLoaded += nodeLoadBuffer.size(); - + // Check that expected ids were allocated assert(actualIds.length == nodeLoadBuffer.size()); for (int i = 0; i < actualIds.length; i++) { - if (nodeLoadBuffer.get(i).id != actualIds[i]) { - logger.warn("Expected ID of node: " + nodeLoadBuffer.get(i).id + + if (nodeLoadBuffer.get(i).id != actualIds[i]) { + logger.warn("Expected ID of node: " + nodeLoadBuffer.get(i).id + " != " + actualIds[i] + " the actual ID"); } } - + nodeLoadBuffer.clear(); - + // convert to microseconds stats.addStats(LinkBenchOp.LOAD_NODE_BULK, timetaken/1000, false); - latencyStats.recordLatency(loaderId, + latencyStats.recordLatency(loaderId, LinkBenchOp.LOAD_NODE_BULK, timetaken); - + if (nodesLoaded >= nextReport) { double totalTimeTaken = (System.currentTimeMillis() - startTime_ms) / 1000.0; logger.debug(String.format( - "Loader #%d: %d/%d nodes loaded at %f nodes/sec", - loaderId, nodesLoaded, totalNodes, - nodesLoaded / totalTimeTaken)); + "Loader #%d: %d/%d nodes loaded at %f nodes/sec", + loaderId, nodesLoaded, totalNodes, + nodesLoaded / totalTimeTaken)); nextReport += REPORT_INTERVAL; } } catch (Throwable e){//Catch exception if any diff --git a/src/java/com/facebook/LinkBench/NodeStore.java b/src/java/com/facebook/LinkBench/NodeStore.java index c8d6fde9..edc922e7 100644 --- a/src/java/com/facebook/LinkBench/NodeStore.java +++ b/src/java/com/facebook/LinkBench/NodeStore.java @@ -27,21 +27,21 @@ public interface NodeStore { // Limit data to 1MB public static final long MAX_NODE_DATA = 1024 * 1024; - + /** initialize the store object */ public void initialize(Properties p, Phase currentPhase, int threadId) throws IOException, Exception; - + /** * Reset node storage to a clean state in shard: * deletes all stored nodes * resets id allocation, with new IDs to be allocated starting from startID */ public void resetNodeStore(String dbid, long startID) throws Exception; - + /** * Adds a new node object to the database. - * + * * This allocates a new id for the object and returns i * @param dbid the db shard to put that object in * @param node a node with all data aside from id filled in. The id @@ -49,7 +49,7 @@ public void initialize(Properties p, * @return the id allocated for the node */ public long addNode(String dbid, Node node) throws Exception; - + /** * Bulk loading to more efficiently load nodes * @param dbid @@ -58,13 +58,13 @@ public void initialize(Properties p, * @throws Exception */ public long[] bulkAddNodes(String dbid, List nodes) throws Exception; - - /** + + /** * Preferred size of data to load * @return */ public int bulkLoadBatchSize(); - + /** * Get a node of the specified type * @param dbid the db shard the id is mapped to @@ -73,17 +73,17 @@ public void initialize(Properties p, * @return null if not found, a Node with all fields filled in otherwise */ public Node getNode(String dbid, int type, long id) throws Exception; - + /** - * Update all parameters of the node specified. + * Update all parameters of the node specified. * @param dbid * @param node * @return true if the update was successful, false if not present */ public boolean updateNode(String dbid, Node node) throws Exception; - + /** - * Delete the object specified by the arguments + * Delete the object specified by the arguments * @param dbid * @param type * @param id @@ -92,7 +92,7 @@ public void initialize(Properties p, public boolean deleteNode(String dbid, int type, long id) throws Exception; public void clearErrors(int loaderId); - + /** * Close the node store and clean up any resources */ diff --git a/src/java/com/facebook/LinkBench/RealDistribution.java b/src/java/com/facebook/LinkBench/RealDistribution.java index 7333fc93..37beb2c6 100644 --- a/src/java/com/facebook/LinkBench/RealDistribution.java +++ b/src/java/com/facebook/LinkBench/RealDistribution.java @@ -34,35 +34,35 @@ public class RealDistribution extends PiecewiseLinearDistribution { public static final String DISTRIBUTION_CONFIG = "realdist"; - private static final Logger logger = - Logger.getLogger(ConfigUtil.LINKBENCH_LOGGER); + private static final Logger logger = + Logger.getLogger(ConfigUtil.LINKBENCH_LOGGER); /* params to shuffler for link degree */ public static final long NLINKS_SHUFFLER_SEED = 20343988438726021L; public static final int NLINKS_SHUFFLER_GROUPS = 1024; - + /* shufflers to generate distributions uncorrelated to above */ public static final long UNCORR_SHUFFLER_SEED = 53238253823453L; public static final int UNCORR_SHUFFLER_GROUPS = 1024; - - /* Shufflers for requests that are correlated with link degree */ + + /* Shufflers for requests that are correlated with link degree */ public static final long WRITE_CORR_SHUFFLER_SEED = NLINKS_SHUFFLER_SEED; public static final int WRITE_CORR_SHUFFLER_GROUPS = NLINKS_SHUFFLER_GROUPS; public static final long READ_CORR_SHUFFLER_SEED = NLINKS_SHUFFLER_SEED; public static final int READ_CORR_SHUFFLER_GROUPS = NLINKS_SHUFFLER_GROUPS; - + /* Shufflers for requests that are uncorrelated with link degree */ public static final long WRITE_UNCORR_SHUFFLER_SEED = UNCORR_SHUFFLER_SEED; public static final int WRITE_UNCORR_SHUFFLER_GROUPS = UNCORR_SHUFFLER_GROUPS; public static final long READ_UNCORR_SHUFFLER_SEED = UNCORR_SHUFFLER_SEED; public static final int READ_UNCORR_SHUFFLER_GROUPS = UNCORR_SHUFFLER_GROUPS; - + public static final long NODE_READ_SHUFFLER_SEED = 4766565305853767165L; public static final int NODE_READ_SHUFFLER_GROUPS = 1024; public static final long NODE_UPDATE_SHUFFLER_SEED = NODE_READ_SHUFFLER_SEED; - public static final int NODE_UPDATE_SHUFFLER_GROUPS = + public static final int NODE_UPDATE_SHUFFLER_GROUPS = NODE_READ_SHUFFLER_GROUPS; public static final long NODE_DELETE_SHUFFLER_SEED = NODE_READ_SHUFFLER_SEED; - public static final int NODE_DELETE_SHUFFLER_GROUPS = + public static final int NODE_DELETE_SHUFFLER_GROUPS = NODE_READ_SHUFFLER_GROUPS; public static enum DistributionType { @@ -77,18 +77,18 @@ public static enum DistributionType { } private DistributionType type = null; - + public RealDistribution() { this.type = null; } - + @Override public void init(long min, long max, Properties props, String keyPrefix) { this.min = min; - this.max = max; + this.max = max; String dist = ConfigUtil.getPropertyRequired(props, keyPrefix + DISTRIBUTION_CONFIG); - + DistributionType configuredType; if (dist.equals("link_reads")) { configuredType = DistributionType.LINK_READS; @@ -104,7 +104,7 @@ public void init(long min, long max, Properties props, String keyPrefix) { throw new RuntimeException("Invalid distribution type for " + "RealDistribution: " + dist); } - + init(props, min, max, configuredType); } @@ -139,7 +139,7 @@ public void init(Properties props, long min, long max, throw new RuntimeException("Unknown distribution type: " + type); } } - + private static ArrayList nlinks_cdf, link_nreads_cdf, link_nwrites_cdf, node_nreads_cdf, node_nwrites_cdf; private static double[] link_nreads_cs, nwrites_cs, node_nreads_cs, node_nwrites_cs; @@ -148,7 +148,7 @@ public void init(Properties props, long min, long max, * the id1 generation, with each cell holding the next id to * return. These are shared between RealDistribution instances * and different threads. - * + * * It is not clear that this works entirely as intended and it * certainly is non-deterministic when multiple threads are * involved. @@ -228,10 +228,10 @@ private static ArrayList readCDF(Scanner scanner) { //convert CDF from ArrayList to Map static NavigableMap getCDF(DistributionType dist) { - ArrayList points = + ArrayList points = dist == DistributionType.LINKS ? nlinks_cdf : dist == DistributionType.LINK_READS? link_nreads_cdf : - dist == DistributionType.LINK_WRITES ? link_nwrites_cdf : + dist == DistributionType.LINK_WRITES ? link_nwrites_cdf : dist == DistributionType.NODE_READS ? node_nreads_cdf : dist == DistributionType.NODE_UPDATES ? node_nwrites_cdf : null; @@ -243,7 +243,7 @@ static NavigableMap getCDF(DistributionType dist) { } return map; } - + /* * This method reads from data_file nlinks, nreads, nwrites discreate * cumulative distribution function (CDF) and produces corresponding @@ -258,7 +258,7 @@ static NavigableMap getCDF(DistributionType dist) { private static void getStatisticalData(Properties props) throws FileNotFoundException { String filename = ConfigUtil.getPropertyRequired(props, Config.DISTRIBUTION_DATA_FILE); - + // If relative path, should be relative to linkbench home directory String fileAbsPath; if (new File(filename).isAbsolute()) { @@ -277,7 +277,7 @@ private static void getStatisticalData(Properties props) throws FileNotFoundExce } logger.info("Loading real distribution data from " + fileAbsPath); - + Scanner scanner = new Scanner(new File(fileAbsPath)); while (scanner.hasNext()) { String type = scanner.next(); @@ -341,7 +341,7 @@ static long getNlinks(long id1, long startid1, long maxid1) { // simple workload balancing return (long)expectedCount(startid1, maxid1, id1, nlinks_cdf); } - + @Override public long choose(Random rng) { if (type == DistributionType.LINKS) { diff --git a/src/java/com/facebook/LinkBench/Shuffler.java b/src/java/com/facebook/LinkBench/Shuffler.java index b50d9c9e..e52e5377 100644 --- a/src/java/com/facebook/LinkBench/Shuffler.java +++ b/src/java/com/facebook/LinkBench/Shuffler.java @@ -16,22 +16,22 @@ package com.facebook.LinkBench; -/* +/* * A class to generate permutation of 0, 1, 2, ..(N-1) using O(1) memory. */ class Shuffler { /* Example to show how algorithm works: n=9; m=4. - * - * Starting with the original a = {0, 1, ..n}, apply the following + * + * Starting with the original a = {0, 1, ..n}, apply the following * transformation to generate a permutation of a: * * 1. Divide 0..9 into multiple groups, each group has length m=4 * a = 0 1 2 3|4 5 6 7|8 9 - * + * * 2. Move elements with same position in each group together * M(a) = 0 4 8|1 5 9|2 6|3 7 - * + * * 3. T(a) = position of i in the permutation in M(a) * T(a) = {0 3 6 8 1 4 7 9 2 5} */ @@ -46,7 +46,7 @@ static long getPermutationValue(long i, long n, long m) { return newgroupid*maxsize + newidx; } else { - return n_maxsize_groups*maxsize + + return n_maxsize_groups*maxsize + (newgroupid - n_maxsize_groups)*minsize + newidx; } } @@ -58,7 +58,7 @@ static long getPermutationValue(long i, long start, long end, long m) { //multiplication of transformations //apply multiple transformation T to make a more random permutation static long getPermutationValue(long i, long start, long end, long[] ms) { - for (int j = 0; j < ms.length; ++j) { + for (int j = 0; j < ms.length; ++j) { i = getPermutationValue(i, start, end, ms[j]); } return i; diff --git a/src/java/com/facebook/LinkBench/Timer.java b/src/java/com/facebook/LinkBench/Timer.java index 2e2511b3..3c06f263 100644 --- a/src/java/com/facebook/LinkBench/Timer.java +++ b/src/java/com/facebook/LinkBench/Timer.java @@ -18,7 +18,7 @@ import java.util.Random; public class Timer { - + /** * Wait an amount of time since the last event determined by the * exponential distribution @@ -34,7 +34,7 @@ public static long waitExpInterval(Random rng, Timer.waitUntil(nextTime_ns); return nextTime_ns; } - + /** * Wait until System.nanoTime() is > the argument * @param time_ns diff --git a/src/java/com/facebook/LinkBench/distributions/AccessDistributions.java b/src/java/com/facebook/LinkBench/distributions/AccessDistributions.java index 94789dba..b90e4408 100644 --- a/src/java/com/facebook/LinkBench/distributions/AccessDistributions.java +++ b/src/java/com/facebook/LinkBench/distributions/AccessDistributions.java @@ -44,7 +44,7 @@ public interface AccessDistribution { * @return */ public abstract long nextID(Random rng, long previousId); - + /** * A shuffler to shuffle the results, or * null if the results shouldn't be shuffled @@ -52,16 +52,16 @@ public interface AccessDistribution { */ public abstract InvertibleShuffler getShuffler(); } - + public static class BuiltinAccessDistribution implements AccessDistribution { private AccessDistMode mode; protected long minid; protected long maxid; private long config; - + /** Use to generate decent quality random longs in range */ - UniformDistribution uniform; - + UniformDistribution uniform; + public BuiltinAccessDistribution(AccessDistMode mode, long minid, long maxid, long config) { this.mode = mode; @@ -76,7 +76,7 @@ public BuiltinAccessDistribution(AccessDistMode mode, public long nextID(Random rng, long previousid) { long newid; double drange = (double)(maxid - minid); - + switch(mode) { case ROUND_ROBIN: //sequential from startid1 to maxid1 (circular) if (previousid <= minid) { @@ -120,12 +120,12 @@ public InvertibleShuffler getShuffler() { return null; } } - + public static class ProbAccessDistribution implements AccessDistribution { private final ProbabilityDistribution dist; - private InvertibleShuffler shuffler; - - public ProbAccessDistribution(ProbabilityDistribution dist, + private InvertibleShuffler shuffler; + + public ProbAccessDistribution(ProbabilityDistribution dist, InvertibleShuffler shuffler) { super(); this.dist = dist; @@ -143,17 +143,17 @@ public InvertibleShuffler getShuffler() { } } - + public static enum AccessDistMode { REAL, // Real empirical distribution ROUND_ROBIN, // Cycle through ids - RECIPROCAL, // Pick with probability + RECIPROCAL, // Pick with probability MULTIPLE, // Pick a multiple of config parameter POWER, // Pick a power of config parameter PERFECT_POWER // Pick a perfect power (square, cube, etc) with exponent // as configured } - + public static AccessDistribution loadAccessDistribution(Properties props, long minid, long maxid, DistributionType kind) throws LinkBenchConfigError { Logger logger = Logger.getLogger(ConfigUtil.LINKBENCH_LOGGER); @@ -183,10 +183,10 @@ public static AccessDistribution loadAccessDistribution(Properties props, default: throw new RuntimeException("Bad kind " + kind); } - + String func_key = keyPrefix + Config.ACCESS_FUNCTION_SUFFIX; String access_func = ConfigUtil.getPropertyRequired(props, func_key); - + try { AccessDistMode mode = AccessDistMode.valueOf(access_func.toUpperCase()); @@ -212,7 +212,7 @@ public static AccessDistribution loadAccessDistribution(Properties props, } /** - * + * * @param className ProbabilityDistribution class name * @param props * @param keyPrefix prefix to use for looking up keys in props diff --git a/src/java/com/facebook/LinkBench/distributions/ApproxHarmonic.java b/src/java/com/facebook/LinkBench/distributions/ApproxHarmonic.java index abf425e8..91901ed6 100644 --- a/src/java/com/facebook/LinkBench/distributions/ApproxHarmonic.java +++ b/src/java/com/facebook/LinkBench/distributions/ApproxHarmonic.java @@ -25,7 +25,7 @@ public class ApproxHarmonic { private static final double EULER_MASCHERONI = 0.5772156649015328606065120900824024310421; - + /** * Approximation to generalized harmonic for 0 >= m >= 1. * Designed to not take more than a couple of seconds to calculate, @@ -55,9 +55,9 @@ public static double generalizedHarmonic(final long n, } else { // Rough approximation for generalized harmonic for // m >= 0 and m <= 1 - + // Standard integral of 1/(n^k) - double integral = (1 / (1 - m)) * Math.pow(n, 1 - m); + double integral = (1 / (1 - m)) * Math.pow(n, 1 - m); // Empirically derived correction factor that is good enough // to get to within 0.2% or so of exact number diff --git a/src/java/com/facebook/LinkBench/distributions/GeometricDistribution.java b/src/java/com/facebook/LinkBench/distributions/GeometricDistribution.java index a7a7da54..ce7c58ef 100644 --- a/src/java/com/facebook/LinkBench/distributions/GeometricDistribution.java +++ b/src/java/com/facebook/LinkBench/distributions/GeometricDistribution.java @@ -25,7 +25,7 @@ /** * Geometric distribution - * + * * NOTE: this generates values in the range [min, max). Since the * real geometric distribution generates values in range [min, inf), * we truncate anything >= max @@ -34,21 +34,21 @@ public class GeometricDistribution implements ProbabilityDistribution { /** The probability parameter that defines the distribution */ private double p = 0.0; - + /** Valid range */ private long min = 0, max = 0; - + private double scale = 0.0; - + public static final String PROB_PARAM_KEY = "prob"; - + @Override public void init(long min, long max, Properties props, String keyPrefix) { double parsedP = ConfigUtil.getDouble(props, keyPrefix + PROB_PARAM_KEY); - + double scaleVal = 1.0;; if (props.containsKey(Config.PROB_MEAN)) { - scaleVal = (max - min) * ConfigUtil.getDouble(props, + scaleVal = (max - min) * ConfigUtil.getDouble(props, keyPrefix + Config.PROB_MEAN); } init(min, max, parsedP, scaleVal); @@ -70,7 +70,7 @@ public double pdf(long id) { public double expectedCount(long id) { return scaledPdf(id, scale); } - + private double scaledPdf(long id, double scaleFactor) { if (id < min || id >= max) return 0.0; long x = id - min; @@ -98,11 +98,11 @@ public long quantile(double r) { * Source: http://www.math.uah.edu/stat/bernoulli/Geometric.html */ if (r == 0.0) return min; // 0.0 must be handled specially - + long x = min + (long)FastMath.ceil( FastMath.log(1 - r) / FastMath.log(1 - p)); // truncate over max - return Math.min(x, max - 1); + return Math.min(x, max - 1); } } diff --git a/src/java/com/facebook/LinkBench/distributions/Harmonic.java b/src/java/com/facebook/LinkBench/distributions/Harmonic.java index 8e153e20..f2bc0ae0 100644 --- a/src/java/com/facebook/LinkBench/distributions/Harmonic.java +++ b/src/java/com/facebook/LinkBench/distributions/Harmonic.java @@ -18,7 +18,7 @@ /** * This code was derived and modified from the Apache Commons * Math 3.0 source release and modified for use in LinkBench - * + * * @author tarmstrong */ import org.apache.commons.math3.util.FastMath; diff --git a/src/java/com/facebook/LinkBench/distributions/ID2Chooser.java b/src/java/com/facebook/LinkBench/distributions/ID2Chooser.java index 3aa6a2bd..719bc1ec 100644 --- a/src/java/com/facebook/LinkBench/distributions/ID2Chooser.java +++ b/src/java/com/facebook/LinkBench/distributions/ID2Chooser.java @@ -41,26 +41,26 @@ public class ID2Chooser { public static final double P_UPDATE_EXIST = 0.9; // Mostly pre-loaded public static final double P_DELETE_EXIST = 0.9; public static final double P_ADD_EXIST = 0.05; // Avoid colliding with pre-loaded too much - + // How many times to try to find a unique id2 private static final int MAX_UNIQ_ITERS = 100; - + private final long startid1; private long maxid1; /** if > 0, choose id2s in range [startid1, randomid2max) */ private final long randomid2max; - + /** Number of distinct link types */ private final int linkTypeCount; - + private final InvertibleShuffler nLinksShuffler; // #links distribution from properties file private final LinkDistribution linkDist; - + // configuration for generating id2 private final int id2gen_config; - + // Information about number of request threads, used to generate // thread-unique id2s private final int nrequesters; @@ -72,29 +72,29 @@ public ID2Chooser(Properties props, long startid1, long maxid1, this.maxid1 = maxid1; this.nrequesters = nrequesters; this.requesterID = requesterID; - + // random number generator for id2 randomid2max = ConfigUtil.getLong(props, Config.RANDOM_ID2_MAX, 0L); - + // configuration for generating id2 id2gen_config = ConfigUtil.getInt(props, Config.ID2GEN_CONFIG, 0); - + linkTypeCount = ConfigUtil.getInt(props, Config.LINK_TYPE_COUNT, 1); linkDist = LinkDistributions.loadLinkDistribution(props, startid1, maxid1); nLinksShuffler = RealDistribution.getShuffler(DistributionType.LINKS, maxid1 - startid1); } - + /** * Choose an ids * @param rng * @param id1 * @param link_type - * @param outlink_ix this is the ith link of this type for this id1 + * @param outlink_ix this is the ith link of this type for this id1 * @return */ - public long chooseForLoad(Random rng, long id1, long link_type, + public long chooseForLoad(Random rng, long id1, long link_type, long outlink_ix) { if (randomid2max == 0) { return id1 + outlink_ix; @@ -102,16 +102,16 @@ public long chooseForLoad(Random rng, long id1, long link_type, return rng.nextInt((int)randomid2max); } } - + /** - * Choose an id2 for an operation given an id1 + * Choose an id2 for an operation given an id1 * @param id1 * @param linkType - * @param pExisting approximate probability that id should be in + * @param pExisting approximate probability that id should be in * existing range * @return */ - public long chooseForOp(Random rng, long id1, long linkType, + public long chooseForOp(Random rng, long id1, long linkType, double pExisting) { long nlinks = calcLinkCount(id1, linkType); long range = calcID2Range(pExisting, nlinks); @@ -145,12 +145,12 @@ public long[] chooseMultipleForOp(Random rng, long id1, long linkType, id2s[i] = id2; } } - + return id2s; } /** - * Check if id2 is in first n elements of id2s + * Check if id2 is in first n elements of id2s * @param id2s * @param i * @param id2 @@ -176,24 +176,24 @@ private long calcID2Range(double pExisting, long nlinks) { /** * Internal helper to choose id * @param rng - * @param id1 + * @param id1 * @param range range size of id2s to select within * @return */ private long chooseForOpInternal(Random rng, long id1, long range) { assert(range >= 1); // We want to sometimes add a link that already exists and sometimes - // add a new link. So generate id2 such that it has roughly pExisting + // add a new link. So generate id2 such that it has roughly pExisting // chance of already existing. - // This happens unless randomid2max is non-zero (in which case just pick a + // This happens unless randomid2max is non-zero (in which case just pick a // random id2 upto randomid2max). - long id2; + long id2; if (randomid2max == 0) { id2 = id1 + rng.nextInt((int)range); } else { id2 = rng.nextInt((int)randomid2max); } - + if (id2gen_config == 1) { return fixId2(id2, nrequesters, requesterID, randomid2max); } else { @@ -202,7 +202,7 @@ private long chooseForOpInternal(Random rng, long id1, long range) { } public boolean sameShuffle; - /** + /** * Calculates the original number of outlinks for a given id1 (i.e. the * number that would have been loaded) * Sets sameShuffle field to true if shuffled was same as original @@ -212,7 +212,7 @@ public long calcTotalLinkCount(long id1) { assert(id1 >= startid1 && id1 < maxid1); // Shuffle. A low id after shuffling means many links, a high means few long shuffled; - if (linkDist.doShuffle()) { + if (linkDist.doShuffle()) { shuffled = startid1 + nLinksShuffler.invertPermute(id1 - startid1); } else { shuffled = id1; @@ -234,7 +234,7 @@ private static long fixId2(long id2, long nrequesters, if ((newid2 > randomid2max) && (randomid2max > 0)) newid2 -= nrequesters; return newid2; } - + public long[] getLinkTypes() { long res[] = new long[linkTypeCount]; // Just have link types in a sequence starting at the default one @@ -243,7 +243,7 @@ public long[] getLinkTypes() { } return res; } - + /** * Choose a link type. * For now just select each type with equal probability. @@ -251,9 +251,9 @@ public long[] getLinkTypes() { public long chooseRandomLinkType(Random rng) { return LinkStore.DEFAULT_LINK_TYPE + rng.nextInt(linkTypeCount); } - + public long calcLinkCount(long id1, long linkType) { - // Divide total links between types so that total is correct + // Divide total links between types so that total is correct long totCount = calcTotalLinkCount(id1); long minCount = totCount / linkTypeCount; long leftOver = totCount - minCount; diff --git a/src/java/com/facebook/LinkBench/distributions/LinkDistributions.java b/src/java/com/facebook/LinkBench/distributions/LinkDistributions.java index aa5ac226..a4596098 100644 --- a/src/java/com/facebook/LinkBench/distributions/LinkDistributions.java +++ b/src/java/com/facebook/LinkBench/distributions/LinkDistributions.java @@ -29,7 +29,7 @@ public class LinkDistributions { public static interface LinkDistribution { public abstract long getNlinks(long id1); - + /** * Let caller know it should shuffle IDs * @return @@ -56,7 +56,7 @@ public boolean doShuffle() { } } - /** + /** * Built-in distributions */ public static enum LinkDistMode { @@ -67,7 +67,7 @@ public static enum LinkDistMode { PERFECT_SQUARES, EXPONENTIAL } - + /** * Some link distributions using arithmetic tricks */ @@ -78,7 +78,7 @@ public static class ArithLinkDistribution implements LinkDistribution { private long nlinks_default; private long minid1, maxid1; - + public ArithLinkDistribution(long minid1, long maxid1, LinkDistMode mode, long nlinks_config, long nlinks_default) { this.minid1 = minid1; @@ -157,7 +157,7 @@ public static LinkDistribution loadLinkDistribution(Properties props, // If not built-in, assume it's a class name return tryDynamicLoad(nlinks_func, props, minid1, maxid1); } - + // real distribution has it own initialization if (mode == LinkDistMode.REAL) { logger.debug("Using real link distribution"); diff --git a/src/java/com/facebook/LinkBench/distributions/LogNormalDistribution.java b/src/java/com/facebook/LinkBench/distributions/LogNormalDistribution.java index a264fed9..a6d91601 100644 --- a/src/java/com/facebook/LinkBench/distributions/LogNormalDistribution.java +++ b/src/java/com/facebook/LinkBench/distributions/LogNormalDistribution.java @@ -27,19 +27,19 @@ public class LogNormalDistribution implements ProbabilityDistribution { private long max; private double mu; // mean of the natural log of random variable private double sigma; // standard deviation of natural log of random variable - + public static final String CONFIG_MEDIAN = "median"; public static final String CONFIG_SIGMA = "sigma"; - + @Override public void init(long min, long max, Properties props, String keyPrefix) { double sigma = ConfigUtil.getDouble(props, CONFIG_SIGMA); double median = ConfigUtil.getDouble(props, CONFIG_MEDIAN); init(min, max, median, sigma); } - + /** - * + * * @param min * @param max * @param median the median value of the distribution @@ -67,7 +67,7 @@ public double expectedCount(long id) { public double cdf(long id) { if (id < min) return 0.0; if (id >= max) return 1.0; - org.apache.commons.math3.distribution.LogNormalDistribution d = + org.apache.commons.math3.distribution.LogNormalDistribution d = new org.apache.commons.math3.distribution.LogNormalDistribution(mu, sigma); return d.cumulativeProbability(id); } @@ -75,11 +75,11 @@ public double cdf(long id) { @Override public long choose(Random rng) { long choice = (long) Math.round(FastMath.exp((rng.nextGaussian() * sigma) + mu)); - if (choice < min) + if (choice < min) return min; - else if (choice >= max) + else if (choice >= max) return max - 1; - else + else return choice; } diff --git a/src/java/com/facebook/LinkBench/distributions/PiecewiseLinearDistribution.java b/src/java/com/facebook/LinkBench/distributions/PiecewiseLinearDistribution.java index 340be1c8..d7bb5d1d 100644 --- a/src/java/com/facebook/LinkBench/distributions/PiecewiseLinearDistribution.java +++ b/src/java/com/facebook/LinkBench/distributions/PiecewiseLinearDistribution.java @@ -22,30 +22,30 @@ /** * A distribution where the cumulative density function is an arbitrary * piecewise linear function. - * - * Rather confusingly there are two possible ways of looking at the + * + * Rather confusingly there are two possible ways of looking at the * distribution. The first is to divide the keyspace by ids, and order * these IDs by the number of accesses. Then DIST-A determines how likely * it is that that given key will be chosen. The second is to divide the * keyspace into buckets, where there are multiple keys in each bucket which * have been accessed the same number of times. There DIST-B determines how - * likely a random key is to fall into each bucket. The input data is + * likely a random key is to fall into each bucket. The input data is * represented as DIST-B, but the probability distribution represented by * this class is DIST-A, so we need to convert from one representation to - * another. + * another. * * The conversion process works as follows. * Suppose you have items numbered 0 to n - 1. Then item i gets assigned * the percentile rank p = i / (n - 1), a number between 0 and 1. - * - * The input is a set of tuples (r, v), where v is the total number of - * observations of the item at percentile p. So the values of the are + * + * The input is a set of tuples (r, v), where v is the total number of + * observations of the item at percentile p. So the values of the are * denominated not in probability density, but rather in number of observation. - * + * * This means that to convert the input to a probability density distribution, * we need to calculate the expected value of the distribution, and then divide * the value by that. - * + * * This is an abstract class: the init method needs to be implemented * @author tarmstrong * @@ -56,17 +56,17 @@ public abstract class PiecewiseLinearDistribution implements ProbabilityDistribu public static class Point implements Comparable { public int value; public double probability; - + public Point(int input_value, double input_probability) { this.value = input_value; this.probability = input_probability; } - + public int compareTo(Point obj) { Point p = (Point)obj; return this.value - p.value; } - + public String toString() { return "(" + value + ", " + probability + ")"; } @@ -79,7 +79,7 @@ protected void init(long min, long max, ArrayList cdf) { long right_points[] = new long[cs.length]; init(min, max, cdf, cs, right_points, expectedValue(cdf)); } - + /** * Init with precalculated values * @param min @@ -98,14 +98,14 @@ protected void init(long min, long max, ArrayList cdf, this.right_points = right_points; this.expected_val = expectedValue; } - + protected long max; protected long min; protected ArrayList cdf; - + protected double[] cs; protected long[] right_points; - + /** * Total number of observations in data */ @@ -123,8 +123,8 @@ public double pdf(long id) { public double expectedCount(long id) { return expectedCount(min, max, id, cdf); } - - public static double expectedCount(long min, long max, long id, + + public static double expectedCount(long min, long max, long id, ArrayList cdf) { if (id < min || id >= max) { return 0.0; @@ -163,10 +163,10 @@ protected static long choose(Random rng, long startid1, long maxid1, double[] cs, long[] right_points) { double max_probability = cs[cs.length - 1]; double p = max_probability * rng.nextDouble(); - + int idx = binarySearch(cs, p); if (idx == 0) idx = 1; - + /* * TODO: this algorithm does not appear to generate data * faithful to the distribution. @@ -188,9 +188,9 @@ protected static long choose(Random rng, long startid1, long maxid1, */ protected static double expectedValue(ArrayList cdf) { // This function is not entirely precise since it assumes - // that the ID space is continuous, which is not an accurate + // that the ID space is continuous, which is not an accurate // approximation for small ID counts - + if (cdf.size() == 0) return 0; // Assume CDF is piecewise linear double sum = 0; @@ -235,10 +235,10 @@ public static int binarySearch(double[] a, double p) { protected static double[] getPDF(ArrayList cdf) { int max_value = cdf.get(cdf.size() - 1).value; double[] pdf = new double[max_value + 1]; - + // set all 0 for (int i = 0; i < pdf.length; ++i) pdf[i] = 0; - + // convert cdf to pdf pdf[cdf.get(0).value] = cdf.get(0).probability; for (int i = 1; i < cdf.size(); ++i) { diff --git a/src/java/com/facebook/LinkBench/distributions/ProbabilityDistribution.java b/src/java/com/facebook/LinkBench/distributions/ProbabilityDistribution.java index 6831b24e..5d2ad777 100644 --- a/src/java/com/facebook/LinkBench/distributions/ProbabilityDistribution.java +++ b/src/java/com/facebook/LinkBench/distributions/ProbabilityDistribution.java @@ -37,16 +37,16 @@ public interface ProbabilityDistribution { * different parameters in properties, this prefix can be * provided to distinguish when looking up keys */ - public abstract void init(long min, long max, + public abstract void init(long min, long max, Properties props, String keyPrefix); - + /** * Probability density function, i.e. P(X = id) * @param id * @return */ public abstract double pdf(long id); - + /** * Probability density function scaled by an implementation-defined * factor (e.g. the number of trials, giving the expected number of values) @@ -54,7 +54,7 @@ public abstract void init(long min, long max, * @return */ public abstract double expectedCount(long id); - + /** * Cumulative distribution function, i.e. for a random variable * X chosen accord to the distribution P(X <= id). @@ -63,7 +63,7 @@ public abstract void init(long min, long max, * @return a probability in range [0.0, 1.0] */ public abstract double cdf(long id); - + /** * Choose a random id in range [min, max) according to the probability * distribution. @@ -71,8 +71,8 @@ public abstract void init(long min, long max, * @return the chosen id */ public abstract long choose(Random rng); - - + + /** * Quantile function for the distribution * @return x such that Pr(X <= x) = p diff --git a/src/java/com/facebook/LinkBench/distributions/UniformDistribution.java b/src/java/com/facebook/LinkBench/distributions/UniformDistribution.java index 145588ad..657e9817 100644 --- a/src/java/com/facebook/LinkBench/distributions/UniformDistribution.java +++ b/src/java/com/facebook/LinkBench/distributions/UniformDistribution.java @@ -27,11 +27,11 @@ * */ public class UniformDistribution implements ProbabilityDistribution { - - private long min = 0; + + private long min = 0; private long max = 1; private double scale = 1.0; - + public void init(long min, long max, Properties props, String keyPrefix) { if (max <= min) { throw new IllegalArgumentException("max = " + max + " <= min = " + min + @@ -40,24 +40,24 @@ public void init(long min, long max, Properties props, String keyPrefix) { this.min = min; this.max = max; if (props != null && props.containsKey(keyPrefix + Config.PROB_MEAN)) { - scale = (max - min) * ConfigUtil.getDouble(props, + scale = (max - min) * ConfigUtil.getDouble(props, keyPrefix + Config.PROB_MEAN); } else { scale = 1.0; } } - + public void init(long min, long max, double scale) { this.min = min; this.max = max; this.scale = scale; } - + @Override public double pdf(long id) { return scaledPDF(id, 1.0); } - + @Override public double expectedCount(long id) { return scaledPDF(id, scale); @@ -84,10 +84,10 @@ public double cdf(long id) { } long n = max - min; long rank = id - min + 1; - + return rank / (double)n; } - + /** * Quantile function */ @@ -98,10 +98,10 @@ public long quantile(double p) { if (i == n) return max - 1; return i + min; } - + // Total number of representable numbers by int private static final long UINT_RANGE = Integer.MAX_VALUE - (long) Integer.MIN_VALUE; - + /** Choose an id X uniformly in the range*/ public long choose(Random rng) { long n = max - min; @@ -112,7 +112,7 @@ public long choose(Random rng) { } else if (n < UINT_RANGE) { return randint2(rng, n); } else { - return UINT_RANGE * rng.nextInt((int)(n / UINT_RANGE)) + + return UINT_RANGE * rng.nextInt((int)(n / UINT_RANGE)) + randint2(rng, n % UINT_RANGE); } } @@ -127,10 +127,10 @@ public long choose(Random rng) { private long randint2(Random rng, long n) { assert(n < UINT_RANGE); double p = Integer.MAX_VALUE / (double)n; - if (rng.nextDouble() < p) { + if (rng.nextDouble() < p) { return rng.nextInt(Integer.MAX_VALUE); } else { - return Integer.MAX_VALUE + + return Integer.MAX_VALUE + (long)(rng.nextInt((int)(n - Integer.MAX_VALUE))); } } diff --git a/src/java/com/facebook/LinkBench/distributions/ZipfDistribution.java b/src/java/com/facebook/LinkBench/distributions/ZipfDistribution.java index 03d1109f..eef26719 100644 --- a/src/java/com/facebook/LinkBench/distributions/ZipfDistribution.java +++ b/src/java/com/facebook/LinkBench/distributions/ZipfDistribution.java @@ -26,51 +26,51 @@ import com.facebook.LinkBench.ConfigUtil; public class ZipfDistribution implements ProbabilityDistribution { - private final Logger logger = Logger.getLogger(ConfigUtil.LINKBENCH_LOGGER); - + private final Logger logger = Logger.getLogger(ConfigUtil.LINKBENCH_LOGGER); + private long min = 0; private long max = 1; private double shape = 0.0; - + /** The total number of items in the world */ private double scale; - + // precomputed values private double alpha = 0.0; private double eta = 0.0; private double zetan = 0.0; private double point5theta = 0.0; - - + + @Override public void init(long min, long max, Properties props, String keyPrefix) { if (max <= min) { throw new IllegalArgumentException("max = " + max + " <= min = " + min + ": probability distribution cannot have zero or negative domain"); } - + this.min = min; this.max = max; String shapeS = props != null ? ConfigUtil.getPropertyRequired(props, keyPrefix + "shape") : null; if (shapeS == null ) { throw new IllegalArgumentException("ZipfDistribution must be provided " + - keyPrefix + "shape parameter"); + keyPrefix + "shape parameter"); } shape = Double.valueOf(shapeS); if (shape <= 0.0) { - throw new IllegalArgumentException("Zipf shape parameter " + shape + + throw new IllegalArgumentException("Zipf shape parameter " + shape + " is not positive"); - + } - + if (props != null && props.containsKey(keyPrefix + Config.PROB_MEAN)) { - scale = (max - min) * ConfigUtil.getDouble(props, + scale = (max - min) * ConfigUtil.getDouble(props, keyPrefix + Config.PROB_MEAN); } else { scale = 1.0; } - + // Precompute some values to speed up future method calls long n = max - min; alpha = 1 / (1 - shape); @@ -80,7 +80,7 @@ public void init(long min, long max, Properties props, String keyPrefix) { point5theta = FastMath.pow(0.5, shape); } - + // For large n, calculating zetan takes a long time. This is a simple // but effective caching technique that speeds up startup a lot @@ -91,19 +91,19 @@ private static class CacheEntry { double shape; double zetan; } - + /** Min value of n to cache */ private static final long MIN_CACHE_VALUE = 1000; private static final int MAX_CACHE_ENTRIES = 1024; - - private static ArrayList zetanCache = + + private static ArrayList zetanCache = new ArrayList(MAX_CACHE_ENTRIES); - + private double calcZetan(long n) { if (n < MIN_CACHE_VALUE) { return uncachedCalcZetan(n); } - + synchronized(ZipfDistribution.class) { for (int i = 0; i < zetanCache.size(); i++) { CacheEntry ce = zetanCache.get(i); @@ -114,7 +114,7 @@ private double calcZetan(long n) { } double calcZetan = uncachedCalcZetan(n); - + synchronized (ZipfDistribution.class) { CacheEntry ce = new CacheEntry(); ce.zetan = calcZetan; @@ -139,10 +139,10 @@ private double uncachedCalcZetan(long n) { // what is happening if (n > 20000000) { logger.info("Precalculating constants for Zipf distribution over " - + n + " items with shape = " + shape + + n + " items with shape = " + shape + ". Please be patient, this can take a little time."); } - + calcZetan = Harmonic.generalizedHarmonic(n, shape); } return calcZetan; @@ -152,19 +152,19 @@ private double uncachedCalcZetan(long n) { public double pdf(long id) { return scaledPDF(id, 1.0); } - + @Override public double expectedCount(long id) { return scaledPDF(id, scale); } - + private double scaledPDF(long id, double scale) { // Calculate this way to avoid losing precision by calculating very // small pdf number if (id < min || id >= max) return 0.0; return (scale / (double) FastMath.pow(id + 1 - min, shape))/ zetan; } - + @Override public double cdf(long id) { if (id < min) return 0.0; @@ -181,7 +181,7 @@ public double cdf(long id) { /** * Algorithm from "Quickly Generating Billion-Record Synthetic Databases", * Gray et. al., 1994 - * + * * Pick a value in range [min, max) according to zipf distribution, * with min being the most likely to be chosen */ @@ -192,7 +192,7 @@ public long choose(Random rng) { /** * Quantile function - * + * * parts of formula are precomputed in init since they are expensive * to calculate and only depend on the distribution parameters */ diff --git a/src/java/com/facebook/LinkBench/generators/DataGenerator.java b/src/java/com/facebook/LinkBench/generators/DataGenerator.java index 0c5eb87c..722aac09 100644 --- a/src/java/com/facebook/LinkBench/generators/DataGenerator.java +++ b/src/java/com/facebook/LinkBench/generators/DataGenerator.java @@ -19,9 +19,9 @@ import java.util.Random; public interface DataGenerator { - - public void init(Properties props, String keyPrefix); - + + public void init(Properties props, String keyPrefix); + /** * Fill the provided array with randomly generated data * @param data diff --git a/src/java/com/facebook/LinkBench/generators/MotifDataGenerator.java b/src/java/com/facebook/LinkBench/generators/MotifDataGenerator.java index e5b53151..69c665e5 100644 --- a/src/java/com/facebook/LinkBench/generators/MotifDataGenerator.java +++ b/src/java/com/facebook/LinkBench/generators/MotifDataGenerator.java @@ -26,23 +26,24 @@ * A simple data generator where the same sequences of bytes, or "motifs" occur * multiple times. This is designed to emulate one particular property of real * data that is exploited by compression algorithms. Typically a short sequence - * of data generated by this generator will not be very compressible on its own, - * as no motifs will recur, but if multiple output strings are concatenated - * together then the same motifs will recur repeatedly and the data will be compressible. - * + * of data generated by this generator will not be very compressible on its own, + * as no motifs will recur, but if multiple output strings are concatenated + * together then the same motifs will recur repeatedly and the data will be + * compressible. + * * The motif data generator has a buffer of "shared" motifs, which reoccur * frequently in the output of the generator - * + * * The data generator generates bytes from within the range of values [min, max). * There is an additional parameter, which is called uniqueness for lack of a * better name. The generator fills a buffer with data in chunks. A chunk * is either generated as random new bytes, or is drawn from the "motifs", - * + * * The uniqueness parameter controls the proportion of new chunks versus duplicated * motifs. It is a probability between 0.0 and 1.0. It can also be seen as the expected * percentage of bytes are generated from scratch. - * - * Control how often motifs appear in data + * + * Control how often motifs appear in data * uniqueness = 0.0: all data drawn from motifs * uniqueness 1.0: completely independent bytes */ @@ -50,32 +51,32 @@ public class MotifDataGenerator implements DataGenerator { private static final int MAX_CHUNK_SIZE = 128; public static final int DEFAULT_MOTIF_BUFFER_SIZE = 512; - - /** Lowest byte to appear in output */ + + /** Lowest byte to appear in output */ private int start; /** Number of distinct bytes to appear in output */ private int range; /** percentage of data drawn from motifs */ private double uniqueness; - - /** - * Buffer with a sequence of random bytes that are + + /** + * Buffer with a sequence of random bytes that are * pasted into output. Starts off null, initialized * on demand. */ private byte motifs[]; /** Size of motif buffer */ private int motifBytes; - + public MotifDataGenerator() { start = '\0'; range = 1; uniqueness = 0.0; } - + /** * Generate characters from start to end (inclusive both ends) * @param start @@ -84,7 +85,7 @@ public MotifDataGenerator() { public void init(int start, int end, double uniqueness) { init(start, end, uniqueness, DEFAULT_MOTIF_BUFFER_SIZE); } - + public void init(int start, int end, double uniqueness, int motifBytes) { if (start < 0 || start >= 256) { throw new LinkBenchConfigError("start " + start + @@ -96,7 +97,7 @@ public void init(int start, int end, double uniqueness, int motifBytes) { } if (start >= end) { - throw new LinkBenchConfigError("startByte " + start + throw new LinkBenchConfigError("startByte " + start + " >= endByte " + end); } this.start = (byte)start; @@ -105,7 +106,7 @@ public void init(int start, int end, double uniqueness, int motifBytes) { this.motifBytes = motifBytes; this.motifs = null; } - + @Override public void init(Properties props, String keyPrefix) { int startByte = ConfigUtil.getInt(props, keyPrefix + @@ -115,7 +116,7 @@ public void init(Properties props, String keyPrefix) { double uniqueness = ConfigUtil.getDouble(props, keyPrefix + Config.MOTIF_GEN_UNIQUENESS); if (props.contains(keyPrefix + Config.MOTIF_GEN_LENGTH)) { - int motifBytes = ConfigUtil.getInt(props, keyPrefix + int motifBytes = ConfigUtil.getInt(props, keyPrefix + Config.MOTIF_GEN_LENGTH); init(startByte, endByte, uniqueness, motifBytes); } else { @@ -124,19 +125,19 @@ public void init(Properties props, String keyPrefix) { } /** - * Give an upper bound for the compression ratio for the algorithm - * @return number between 0.0 and 1.0 - 0.0 is perfectly compressible, + * Give an upper bound for the compression ratio for the algorithm + * @return number between 0.0 and 1.0 - 0.0 is perfectly compressible, * 1.0 is incompressible */ public double estMaxCompression() { // Avg bytes required to represent each character (uniformly distributed) double charCompression = range / (double) 255; // random data shouldn't have any inter-character correlations that can - // be compressed. Upper bound derived by assuming motif is completely - // compressible + // be compressed. Upper bound derived by assuming motif is completely + // compressible return charCompression * uniqueness; } - + @Override public byte[] fill(Random rng, byte[] data) { // Fill motifs now so that we can use rng @@ -146,10 +147,10 @@ public byte[] fill(Random rng, byte[] data) { motifs[i] = (byte) (start + rng.nextInt(range)); } } - + int n = data.length; int chunk = Math.min(MAX_CHUNK_SIZE, motifBytes); - + for (int i = 0; i < n; i += chunk) { if (rng.nextDouble() < uniqueness) { int chunkEnd = Math.min(n, i + chunk); diff --git a/src/java/com/facebook/LinkBench/generators/UniformDataGenerator.java b/src/java/com/facebook/LinkBench/generators/UniformDataGenerator.java index 2f960034..d339a29d 100644 --- a/src/java/com/facebook/LinkBench/generators/UniformDataGenerator.java +++ b/src/java/com/facebook/LinkBench/generators/UniformDataGenerator.java @@ -25,9 +25,9 @@ /** * A super simple data generator that generates a string of * characters chosen uniformly from a range. - * + * * This probably isn't a good generator to use if you want something realistic, - * especially if compressibility properties of the data will affect your + * especially if compressibility properties of the data will affect your * experiment. */ public class UniformDataGenerator implements DataGenerator { @@ -38,7 +38,7 @@ public UniformDataGenerator() { start = '\0'; range = 1; } - + /** * Generate characters from start to end (inclusive both ends) * @param start @@ -55,13 +55,13 @@ public void init(int start, int end) { } if (start >= end) { - throw new LinkBenchConfigError("startByte " + start + throw new LinkBenchConfigError("startByte " + start + " >= endByte " + end); } this.start = (byte)start; this.range = end - start + 1; } - + @Override public void init(Properties props, String keyPrefix) { int startByte = ConfigUtil.getInt(props, keyPrefix + @@ -75,7 +75,7 @@ public void init(Properties props, String keyPrefix) { public byte[] fill(Random rng, byte[] data) { return gen(rng, data, start, range); } - + public static byte[] gen(Random rng, byte[] data, int startByte, int range) { int n = data.length; diff --git a/src/java/com/facebook/LinkBench/stats/LatencyStats.java b/src/java/com/facebook/LinkBench/stats/LatencyStats.java index 788b32cb..f5c063fc 100644 --- a/src/java/com/facebook/LinkBench/stats/LatencyStats.java +++ b/src/java/com/facebook/LinkBench/stats/LatencyStats.java @@ -26,17 +26,17 @@ /** - * Class used to track and compute latency statistics, particularly - * percentiles. Times are divided into buckets, with counts maintained + * Class used to track and compute latency statistics, particularly + * percentiles. Times are divided into buckets, with counts maintained * per bucket. The division into buckets is based on typical latencies * for database operations: most are in the range of 0.1ms to 100ms. * we have 0.1ms-granularity buckets up to 1ms, then 1ms-granularity from - * 1-100ms, then 100ms-granularity, and then 1s-granularity. + * 1-100ms, then 100ms-granularity, and then 1s-granularity. */ public class LatencyStats { public static int MAX_MILLIS = 100; - + /** * Keep track of running mean per thread and op type */ @@ -54,32 +54,32 @@ public LatencyStats(int maxThreads) { bucketCounts = new long[maxThreads][LinkStore.MAX_OPTYPES][NUM_BUCKETS]; maxLatency = new long[maxThreads][LinkStore.MAX_OPTYPES]; } - - + + private static final int SUB_MS_BUCKETS = 10; // Sub-ms granularity private static final int MS_BUCKETS = 99; // ms-granularity buckets private static final int HUNDREDMS_BUCKETS = 9; // 100ms-granularity buckets private static final int SEC_BUCKETS = 9; // 1s-granularity buckets public static final int NUM_BUCKETS = SUB_MS_BUCKETS + MS_BUCKETS + HUNDREDMS_BUCKETS + SEC_BUCKETS + 1; - + /** Counts of operations falling into each bucket */ - private final long bucketCounts[][][]; - + private final long bucketCounts[][][]; + /** Counts of samples per type */ private long sampleCounts[]; - + /** Cumulative bucket counts keyed by type, bucket# (calculated at end) */ - private long bucketCountsCumulative[][]; - + private long bucketCountsCumulative[][]; + /** Maximum latency by thread and type */ private long maxLatency[][]; - + public static int latencyToBucket(long microTime) { long ms = 1000; long msTime = microTime / ms; // Floored if (msTime == 0) { - // Bucket per 0.1 ms + // Bucket per 0.1 ms return (int) (microTime / 100); } else if (msTime < 100) { // msBucket = 0 means 1-2 ms @@ -96,9 +96,9 @@ public static int latencyToBucket(long microTime) { return NUM_BUCKETS - 1; } } - + /** - * + * * @param bucket * @return inclusive min and exclusive max time in microsecs for bucket */ @@ -126,10 +126,10 @@ public static long[] bucketBound(int bucket) { } return res; } - - /** - * Used by the linkbench driver to record latency of each + + /** + * Used by the linkbench driver to record latency of each * individual call */ public void recordLatency(int threadid, LinkBenchOp type, @@ -137,14 +137,14 @@ public void recordLatency(int threadid, LinkBenchOp type, long opBuckets[] = bucketCounts[threadid][type.ordinal()]; int bucket = latencyToBucket(microtimetaken); opBuckets[bucket]++; - + double time_ms = microtimetaken / 1000.0; if (means[threadid][type.ordinal()] == null) { means[threadid][type.ordinal()] = new RunningMean(time_ms); } else { means[threadid][type.ordinal()].addSample(time_ms); } - + if (maxLatency[threadid][type.ordinal()] < microtimetaken) { maxLatency[threadid][type.ordinal()] = microtimetaken; } @@ -163,7 +163,7 @@ public void displayLatencyStats() { if (sampleCounts[type.ordinal()] == 0) { // no samples of this type continue; } - + DecimalFormat df = new DecimalFormat("#.###"); // Format to max 3 decimal place logger.info(type.displayName() + " count = " + sampleCounts[type.ordinal()] + " " + @@ -180,10 +180,10 @@ public void displayLatencyStats() { public void printCSVStats(PrintStream out, boolean header) { printCSVStats(out, header, LinkBenchOp.values()); } - + public void printCSVStats(PrintStream out, boolean header, LinkBenchOp... ops) { int percentiles[] = new int[] {25, 50, 75, 95, 99}; - + // Write out the header if (header) { out.print("op,count"); @@ -193,10 +193,10 @@ public void printCSVStats(PrintStream out, boolean header, LinkBenchOp... ops) { out.print(",max,mean"); out.println(); } - + // Print in milliseconds down to 10us granularity DecimalFormat df = new DecimalFormat("#.##"); - + for (LinkBenchOp op: ops) { long samples = sampleCounts[op.ordinal()]; if (samples == 0) { @@ -205,7 +205,7 @@ public void printCSVStats(PrintStream out, boolean header, LinkBenchOp... ops) { out.print(op.name()); out.print(","); out.print(samples); - + for (int percentile: percentiles) { long bounds[] = getBucketBounds(op, percentile); out.print(","); @@ -213,7 +213,7 @@ public void printCSVStats(PrintStream out, boolean header, LinkBenchOp... ops) { out.print(","); out.print(df.format(bounds[1] / 1000.0)); } - + out.print(","); out.print(df.format(getMax(op))); out.print(","); @@ -221,7 +221,7 @@ public void printCSVStats(PrintStream out, boolean header, LinkBenchOp... ops) { out.println(); } } - + /** * Fill in the counts and means arrays */ @@ -236,7 +236,7 @@ private void calcMeans() { } } sampleCounts[i] = samples; - + double weightedMean = 0.0; for (int thread = 0; thread < maxThreads; thread++) { if (means[thread][i] != null) { @@ -246,13 +246,13 @@ private void calcMeans() { } finalMeans[i] = weightedMean; } - + } private void calcCumulativeBuckets() { // Calculate the cumulative operation counts by bucket for each type bucketCountsCumulative = new long[LinkStore.MAX_OPTYPES][NUM_BUCKETS]; - + for (int type = 0; type < LinkStore.MAX_OPTYPES; type++) { long count = 0; for (int bucket = 0; bucket < NUM_BUCKETS; bucket++) { @@ -267,8 +267,8 @@ private void calcCumulativeBuckets() { private long[] getBucketBounds(LinkBenchOp type, long percentile) { long n = sampleCounts[type.ordinal()]; // neededRank is the rank of the sample at the desired percentile - long neededRank = (long) ((percentile / 100.0) * n); - + long neededRank = (long) ((percentile / 100.0) * n); + int bucketNum = -1; for (int i = 0; i < NUM_BUCKETS; i++) { long rank = bucketCountsCumulative[type.ordinal()][i]; @@ -279,14 +279,14 @@ private long[] getBucketBounds(LinkBenchOp type, long percentile) { } } assert(bucketNum >= 0); // Should definitely be found; - - + + return bucketBound(bucketNum); } - - + + /** - * + * * @return A human-readable string for the bucket bounds */ private String percentileString(LinkBenchOp type, long percentile) { @@ -296,7 +296,7 @@ private String percentileString(LinkBenchOp type, long percentile) { static String boundsToString(long[] bucketBounds) { double minMs = bucketBounds[0] / 1000.0; double maxMs = bucketBounds[1] / 1000.0; - + DecimalFormat df = new DecimalFormat("#.##"); // Format to max 1 decimal place return "["+ df.format(minMs) + "," + df.format(maxMs) + "]"; } diff --git a/src/java/com/facebook/LinkBench/stats/RunningMean.java b/src/java/com/facebook/LinkBench/stats/RunningMean.java index 5c896582..b5880be0 100644 --- a/src/java/com/facebook/LinkBench/stats/RunningMean.java +++ b/src/java/com/facebook/LinkBench/stats/RunningMean.java @@ -15,9 +15,9 @@ */ package com.facebook.LinkBench.stats; -/** +/** * Keep track of mean in numerically stable way - * See "Comparison of Several Algorithms for Computing Sample Means and + * See "Comparison of Several Algorithms for Computing Sample Means and * Variances", Ling, 1974, J. American Stat. Assoc. */ public class RunningMean { @@ -26,10 +26,10 @@ public class RunningMean { /** First sample */ private final double v1; - + /** sum of difference */ private double running; - + /** initialize with first sample */ public RunningMean(double v1) { super(); @@ -42,7 +42,7 @@ public void addSample(double vi) { n++; running += (vi - v1); } - + public double mean() { return v1 + running / n; } @@ -50,4 +50,4 @@ public double mean() { public long samples() { return n; } -} \ No newline at end of file +} diff --git a/src/java/com/facebook/LinkBench/stats/SampledStats.java b/src/java/com/facebook/LinkBench/stats/SampledStats.java index 7514721d..0fcb40e1 100644 --- a/src/java/com/facebook/LinkBench/stats/SampledStats.java +++ b/src/java/com/facebook/LinkBench/stats/SampledStats.java @@ -33,10 +33,10 @@ * This class is used to keep track of statistics. It collects a sample of the * total data (controlled by maxsamples) and can then compute statistics based * on that sample. - * + * * The overall idea is to compute reasonably accurate statistics using bounded * space. - * + * * Currently the class is used to print out stats at given intervals, with the * sample taken over a given time interval and printed at the end of the interval. */ @@ -51,7 +51,7 @@ public class SampledStats { // samples for various optypes private long[][] samples; - /** Number of operations that the sample is drawn from */ + /** Number of operations that the sample is drawn from */ private int opsSinceReset[]; // minimums encounetered per operation type @@ -62,15 +62,15 @@ public class SampledStats { // #errors encountered per type private long errors[]; - + // Displayed along with stats private int threadID; - + private final Logger logger = Logger.getLogger(ConfigUtil.LINKBENCH_LOGGER); /** Stream to write csv output to ( null if no csv output ) */ private final PrintStream csvOutput; - + /** Random number generator used to decide which to include in sample */ private Random rng; @@ -85,9 +85,9 @@ public SampledStats(int input_threadID, maximums = new long[LinkStore.MAX_OPTYPES]; numops = new long[LinkStore.MAX_OPTYPES]; errors = new long[LinkStore.MAX_OPTYPES]; - + rng = new Random(); - + csvOutput = null; } @@ -117,7 +117,7 @@ public void addStats(LinkBenchOp type, long timetaken, boolean error) { double pReplace = ((double)maxsamples) / opIndex; if (rng.nextDouble() < pReplace) { // Select sample to replace randomly - samples[type.ordinal()][rng.nextInt(maxsamples)] = timetaken; + samples[type.ordinal()][rng.nextInt(maxsamples)] = timetaken; } } } @@ -142,17 +142,17 @@ private void displayStats(LinkBenchOp type, int start, int end, int elems = end - start; long timestamp = nowTime_ms / 1000; long sampleDuration = nowTime_ms - sampleStartTime_ms; - + if (elems <= 0) { logger.info("ThreadID = " + threadID + " " + type.displayName() + " totalops = " + numops[type.ordinal()] + " totalErrors = " + errors[type.ordinal()] + - " ops = " + opsSinceReset[type.ordinal()] + + " ops = " + opsSinceReset[type.ordinal()] + " sampleDuration = " + sampleDuration + "ms" + " samples = " + elems); if (csvOutput != null) { - csvOutput.println(threadID + "," + timestamp + "," + type.name() + + csvOutput.println(threadID + "," + timestamp + "," + type.name() + "," + numops[type.ordinal()] + "," + errors[type.ordinal()] + "," + 0 + "," + sampleDuration + ",0,,,,,,,,,"); @@ -167,7 +167,7 @@ private void displayStats(LinkBenchOp type, int start, int end, for (int i = start + 1; i < end; i++) { meanCalc.addSample(samples[type.ordinal()][i]); } - + long min = samples[type.ordinal()][start]; long p25 = samples[type.ordinal()][start + elems/4]; long p50 = samples[type.ordinal()][start + elems/2]; @@ -177,8 +177,8 @@ private void displayStats(LinkBenchOp type, int start, int end, long p99 = samples[type.ordinal()][end - 1 - elems/100]; long max = samples[type.ordinal()][end - 1]; double mean = meanCalc.mean(); - - + + DecimalFormat df = new DecimalFormat("#.##"); logger.info("ThreadID = " + threadID + " " + type.displayName() + @@ -197,16 +197,16 @@ private void displayStats(LinkBenchOp type, int start, int end, " 99% = " + p99 + " max = " + max); if (csvOutput != null) { - csvOutput.println(threadID + "," + timestamp + "," + type.name() + + csvOutput.println(threadID + "," + timestamp + "," + type.name() + "," + numops[type.ordinal()] + "," + errors[type.ordinal()] + "," + opsSinceReset[type.ordinal()] + "," + sampleDuration + "," + elems + "," + mean + "," + min + "," + p25 + "," + p50 + "," + p75 + "," + p90 + "," + p95 + "," + p99 + "," + max); } } - + /** - * Write a header with column names for a csv file showing progress + * Write a header with column names for a csv file showing progress * @param out */ public static void writeCSVHeader(PrintStream out) { @@ -219,7 +219,7 @@ public void displayStatsAll(long sampleStartTime_ms, long nowTime_ms) { displayStats(sampleStartTime_ms, nowTime_ms, Arrays.asList(LinkBenchOp.values())); } - + public void displayStats(long sampleStartTime_ms, long nowTime_ms, Collection ops) { for (LinkBenchOp op: ops) { diff --git a/src/java/com/facebook/LinkBench/util/ClassLoadUtil.java b/src/java/com/facebook/LinkBench/util/ClassLoadUtil.java index 635e77d3..f20ab530 100644 --- a/src/java/com/facebook/LinkBench/util/ClassLoadUtil.java +++ b/src/java/com/facebook/LinkBench/util/ClassLoadUtil.java @@ -23,14 +23,14 @@ public class ClassLoadUtil { private static final Class[] EMPTY_ARRAY = new Class[]{}; - + /** * Load a class by name. * @param name the class name. * @return the class object. * @throws ClassNotFoundException if the class is not found. */ - public static Class getClassByName(String name) + public static Class getClassByName(String name) throws ClassNotFoundException { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); return Class.forName(name, true, classLoader); @@ -47,7 +47,7 @@ public static T newInstance(Class theClass, Class expected) { try { if (!expected.isAssignableFrom(theClass)) { throw new Exception("Specified class " + theClass.getName() + "" + - "does not extend/implement " + expected.getName()); + "does not extend/implement " + expected.getName()); } Class clazz = (Class)theClass; Constructor meth = clazz.getDeclaredConstructor(EMPTY_ARRAY); @@ -58,7 +58,7 @@ public static T newInstance(Class theClass, Class expected) { } return result; } - + public static T newInstance(String className, Class expected) throws ClassNotFoundException { return newInstance(getClassByName(className), expected); diff --git a/src/java/com/facebook/rocks/swift/Code.java b/src/java/com/facebook/rocks/swift/Code.java index 9588b583..54402718 100644 --- a/src/java/com/facebook/rocks/swift/Code.java +++ b/src/java/com/facebook/rocks/swift/Code.java @@ -18,4 +18,4 @@ public int getValue() { return value; } -} \ No newline at end of file +} diff --git a/src/java/com/facebook/rocks/swift/CompressionType.java b/src/java/com/facebook/rocks/swift/CompressionType.java index 373d4fe7..39fca4da 100644 --- a/src/java/com/facebook/rocks/swift/CompressionType.java +++ b/src/java/com/facebook/rocks/swift/CompressionType.java @@ -18,4 +18,4 @@ public int getValue() { return value; } -} \ No newline at end of file +} diff --git a/src/java/com/facebook/rocks/swift/IOError.java b/src/java/com/facebook/rocks/swift/IOError.java index 90da9347..08be74fa 100644 --- a/src/java/com/facebook/rocks/swift/IOError.java +++ b/src/java/com/facebook/rocks/swift/IOError.java @@ -15,4 +15,4 @@ public class IOError extends Exception @ThriftField(value=1, name="message") public void setMessage(final String message) { this.message = message; } -} \ No newline at end of file +} diff --git a/src/java/com/facebook/rocks/swift/ReadOptions.java b/src/java/com/facebook/rocks/swift/ReadOptions.java index ed5c1ec8..b67c20d0 100644 --- a/src/java/com/facebook/rocks/swift/ReadOptions.java +++ b/src/java/com/facebook/rocks/swift/ReadOptions.java @@ -29,4 +29,4 @@ public class ReadOptions @ThriftField(value=3, name="snapshot") public void setSnapshot(final Snapshot snapshot) { this.snapshot = snapshot; } -} \ No newline at end of file +} diff --git a/src/java/com/facebook/rocks/swift/ResultSnapshot.java b/src/java/com/facebook/rocks/swift/ResultSnapshot.java index ed5d2e36..5e19d43b 100644 --- a/src/java/com/facebook/rocks/swift/ResultSnapshot.java +++ b/src/java/com/facebook/rocks/swift/ResultSnapshot.java @@ -21,4 +21,4 @@ public class ResultSnapshot @ThriftField(value=2, name="snapshot") public void setSnapshot(final Snapshot snapshot) { this.snapshot = snapshot; } -} \ No newline at end of file +} diff --git a/src/java/com/facebook/rocks/swift/RocksIterateResponse.java b/src/java/com/facebook/rocks/swift/RocksIterateResponse.java index 1cf22200..f0e6906f 100644 --- a/src/java/com/facebook/rocks/swift/RocksIterateResponse.java +++ b/src/java/com/facebook/rocks/swift/RocksIterateResponse.java @@ -21,4 +21,4 @@ public class RocksIterateResponse @ThriftField(value=2, name="data") public void setData(final List data) { this.data = data; } -} \ No newline at end of file +} diff --git a/src/java/com/facebook/rocks/swift/Snapshot.java b/src/java/com/facebook/rocks/swift/Snapshot.java index 5ed2bc74..16f95f16 100644 --- a/src/java/com/facebook/rocks/swift/Snapshot.java +++ b/src/java/com/facebook/rocks/swift/Snapshot.java @@ -13,4 +13,4 @@ public class Snapshot @ThriftField(value=1, name="snapshotid") public void setSnapshotid(final long snapshotid) { this.snapshotid = snapshotid; } -} \ No newline at end of file +} diff --git a/src/java/com/facebook/rocks/swift/WriteOptions.java b/src/java/com/facebook/rocks/swift/WriteOptions.java index 52e5fb40..23bcbee5 100644 --- a/src/java/com/facebook/rocks/swift/WriteOptions.java +++ b/src/java/com/facebook/rocks/swift/WriteOptions.java @@ -21,4 +21,4 @@ public class WriteOptions @ThriftField(value=2, name="disableWAL") public void setDisableWAL(final boolean disableWAL) { this.disableWAL = disableWAL; } -} \ No newline at end of file +} diff --git a/src/test/java/com/facebook/LinkBench/DistributionTestBase.java b/src/test/java/com/facebook/LinkBench/DistributionTestBase.java index 9c0e2ff8..18c80d28 100644 --- a/src/test/java/com/facebook/LinkBench/DistributionTestBase.java +++ b/src/test/java/com/facebook/LinkBench/DistributionTestBase.java @@ -26,34 +26,34 @@ /** * This test implements generic unit tests for different implementations of - * ProbabilityDistribution. - * + * ProbabilityDistribution. + * * Most of these tests are either sanity tests (check that output is within * expected range and obeys basic invariants), and consistency tests * (check that the output of two different methods is consistent). - * + * * While these tests go a long way to checking the consistency of the * behavior of the ProbabilityDistribution, it cannot check that the * specific correct values are generated: it is helpful to implement * additional tests for each concrete implementation. - * + * * @author tarmstrong */ public abstract class DistributionTestBase extends TestCase { protected abstract ProbabilityDistribution getDist(); - + protected Properties getDistParams() { return new Properties(); } - + /** Number of cdf checks to perform */ protected int cdfChecks() { return 50000; } - + /** Number of pdf checks */ - protected int pdfChecks() { + protected int pdfChecks() { return 50000; } @@ -61,7 +61,7 @@ protected Bucketer getBucketer() { return new UniformBucketer(cdfChecks()); } - /** Percentage difference between cdf and choose() to tolerate */ + /** Percentage difference between cdf and choose() to tolerate */ protected double tolerance() { return 0.002; } @@ -71,7 +71,7 @@ public Random initRandom(String testName) { System.err.println("Choose seed " + seed + " for test " + testName); return new Random(seed); } - + /** * Check a few invariants cdf should adhere to */ @@ -80,12 +80,12 @@ public void testCDFSanity() { ProbabilityDistribution dist = getDist(); long min = 453, max = 26546454; dist.init(min, max, getDistParams(), ""); - + assertEquals(dist.cdf(min-1), 0.0); assertEquals(dist.cdf(min-234321), 0.0); assertEquals(dist.cdf(max), 1.0); assertEquals(dist.cdf(max+2343242224234L), 1.0); - + // Check cdf is monotonically increasing double last = 0.0; long step = (max - min) / cdfChecks(); @@ -95,24 +95,24 @@ public void testCDFSanity() { last = p; } } - + @Test public void testPDFSanity() { ProbabilityDistribution dist = getDist(); long min = 453, max = 26546454; dist.init(min, max, getDistParams(), ""); - + assertEquals(0.0, dist.pdf(min-1)); assertEquals(0.0, dist.pdf(min-234321)); assertEquals(0.0, dist.pdf(max)); assertEquals(0.0, dist.pdf(max+2343242224234L)); - + // Check pdf is in correct range double total = 0.0; long step = (max - min) / pdfChecks(); for (long id = min; id < max; id += step) { double p = dist.pdf(id); - if ((id - min) < step * 100) { + if ((id - min) < step * 100) { System.err.println("p(X=" + id + ") = " + p); } assertTrue(p >= 0.0); @@ -121,13 +121,13 @@ public void testPDFSanity() { } assert(total <= 1.0); } - + @Test public void testPDFSum() { ProbabilityDistribution dist = getDist(); long min = 1, max = 50; dist.init(min, max, getDistParams(), ""); - + // Check sum of pdf over small range // Order of least to most probably to minimize sum error double total = 0.0; @@ -156,25 +156,25 @@ public void testChooseSanity() { assertTrue(id < max); } } - + /** * Check that choose() and cdf() are returning consistent results * (i.e. that the result of choose are distributed according to cdf) */ @Test public void testCDFChooseConsistency() { - + long min = 100, max = 100000; Bucketer bucketer = getBucketer(); int bucketCount = bucketer.getBucketCount(); int buckets[] = new int[bucketCount]; - + long n = max - min; - + Random rng = initRandom("testCDFChooseConsistency"); ProbabilityDistribution dist = getDist(); dist.init(min, max, getDistParams(), ""); - + int trials = 1000000; for (int i = 0; i < trials; i++) { long id = dist.choose(rng); @@ -182,7 +182,7 @@ public void testCDFChooseConsistency() { int bucket = bucketer.chooseBucket(off, n); buckets[bucket]++; } - + int totalCount = 0; boolean fail = false; for (int b = 0; b < bucketCount; b++) { @@ -193,29 +193,29 @@ public void testCDFChooseConsistency() { // 0.2% error if (Math.abs(expCDF - actCDF) > tolerance()) { System.err.println(String.format("Divergence between CDF and " + - "choose function: P(X <= %d) act: %f exp: %f", bucketTop, - actCDF, expCDF)); + "choose function: P(X <= %d) act: %f exp: %f", bucketTop, + actCDF, expCDF)); fail = true; } } if (fail) { fail("Divergence between cdf and choose methods: see preceding output " + - "for details"); + "for details"); } } - + @Test public void testCDFPDFConsistency() { long min = 252352, max = 6544543; ProbabilityDistribution dist = getDist(); dist.init(min, max, getDistParams(), ""); - + long step = (max - min) / cdfChecks(); for (long id = min + 1; id < max; id += step) { double c = dist.cdf(id); double c1 = dist.cdf(id - 1); double p = dist.pdf(id); - + double err = Math.abs((c - c1) - p); if (err > 0.0001) { fail(String.format("Error > 0.001: cdf(%d) - cdf(%d) = %f, pdf(%d) = %f", @@ -223,13 +223,13 @@ public void testCDFPDFConsistency() { } } } - + @Test public void testQuantileSanity() { long min = 0, max = 1000; ProbabilityDistribution dist = getDist(); dist.init(min, max, getDistParams(), ""); - + long last = dist.quantile(0.0); for (double q = 0.0; q <= 1.0; q += 0.125) { long id = dist.quantile(q); @@ -239,7 +239,7 @@ public void testQuantileSanity() { assertTrue(id >= last); last = id; } - + // min should be most probable, and therefore should definitely // be returned by quantile assertEquals(min, dist.quantile(0.0)); @@ -248,7 +248,7 @@ public void testQuantileSanity() { /** * Different distributions should be bucketed in different ways * to test their fit. For example, the zipf distribution treats - * lower keys specially so we want to have better resolution for + * lower keys specially so we want to have better resolution for * those */ static interface Bucketer { @@ -256,10 +256,10 @@ static interface Bucketer { public int chooseBucket(long i, long n); public long bucketMax(int bucket, long n); } - + static class UniformBucketer implements Bucketer { final int bucketCount; - + public UniformBucketer(int bucketCount) { this.bucketCount = bucketCount; } @@ -267,13 +267,13 @@ public UniformBucketer(int bucketCount) { public int getBucketCount() { return bucketCount; } - + public int chooseBucket(long i, long n) { return (int)((i * bucketCount) / n); } - + public long bucketMax(int bucket, long n) { - return ((long)((((double)bucket+1)/bucketCount)*n)) - 1; + return ((long)((((double)bucket+1)/bucketCount)*n)) - 1; } } } diff --git a/src/test/java/com/facebook/LinkBench/DummyLinkStore.java b/src/test/java/com/facebook/LinkBench/DummyLinkStore.java index 449478f1..112eefe8 100644 --- a/src/test/java/com/facebook/LinkBench/DummyLinkStore.java +++ b/src/test/java/com/facebook/LinkBench/DummyLinkStore.java @@ -32,16 +32,16 @@ public class DummyLinkStore extends GraphStore { public LinkStore wrappedStore; - public GraphStore wrappedGraphStore; - + public GraphStore wrappedGraphStore; + public DummyLinkStore() { this(null); } - + public DummyLinkStore(LinkStore wrappedStore) { this(wrappedStore, false); } - + public DummyLinkStore(LinkStore wrappedStore, boolean alreadyInitialized) { this.wrappedStore = wrappedStore; if (wrappedStore instanceof GraphStore) { @@ -49,23 +49,23 @@ public DummyLinkStore(LinkStore wrappedStore, boolean alreadyInitialized) { } this.initialized = alreadyInitialized; } - + /** * @return true if real data is written and can be queried */ public boolean isRealLinkStore() { return wrappedStore != null; } - + /** * @return true if real node data is written and can be queried */ public boolean isRealGraphStore() { return wrappedGraphStore != null; } - + public boolean initialized = false; - + public long adds = 0; public long deletes = 0; public long updates = 0; @@ -74,7 +74,7 @@ public boolean isRealGraphStore() { public long getLinkLists = 0; public long getLinkListsHistory = 0; public long countLinks = 0; - + public long addNodes = 0; public long updateNodes = 0; public long deleteNodes = 0; @@ -85,7 +85,7 @@ public boolean isRealGraphStore() { public long bulkLoadLinkRows; public long bulkLoadCountOps; public long bulkLoadCountRows; - + @Override public void initialize(Properties p, Phase currentPhase, int threadId) throws IOException, Exception { diff --git a/src/test/java/com/facebook/LinkBench/DummyLinkStoreTest.java b/src/test/java/com/facebook/LinkBench/DummyLinkStoreTest.java index 329344da..825ed826 100644 --- a/src/test/java/com/facebook/LinkBench/DummyLinkStoreTest.java +++ b/src/test/java/com/facebook/LinkBench/DummyLinkStoreTest.java @@ -29,7 +29,8 @@ protected void initStore(Properties props) { } @Override - protected DummyLinkStore getStoreHandle(boolean initialized) throws IOException, Exception { + protected DummyLinkStore getStoreHandle(boolean initialized) + throws IOException, Exception { DummyLinkStore store = new DummyLinkStore(); if (initialized) { store.initialize(props, Phase.REQUEST, 0); diff --git a/src/test/java/com/facebook/LinkBench/GeneratedDataDump.java b/src/test/java/com/facebook/LinkBench/GeneratedDataDump.java index d709d745..4de75eaf 100644 --- a/src/test/java/com/facebook/LinkBench/GeneratedDataDump.java +++ b/src/test/java/com/facebook/LinkBench/GeneratedDataDump.java @@ -28,13 +28,13 @@ /** * Generate some sample data using data generators, in order to test out compressibility * of randomly generated data. - * + * * This generates several output files consistent of many generated payload data fields * separated by newlines */ public class GeneratedDataDump { private static final Random rng = new Random(); - + public static void main(String args[]) { String outputDir = ""; if (args.length == 0) { @@ -45,8 +45,8 @@ public static void main(String args[]) { System.err.println("GeneratedDataDump "); System.exit(1); } - - + + // Number of bytes per row final int objBytes = 256; final int assocBytes = 6; @@ -68,10 +68,10 @@ private static void writeGeneratedDataFile(String outFileName, System.err.println("file " + outFileName + " could not be opened"); System.exit(1); } - - + + byte buf[] = new byte[bytes]; - + try { for (int i = 0; i < rows; i++) { gen.fill(rng, buf); @@ -99,7 +99,7 @@ private static MotifDataGenerator makeMotifObj() { gen.init(start, end, uniqueness); return gen; } - + private static DataGenerator makeUniformAssoc() { UniformDataGenerator gen = new UniformDataGenerator(); gen.init(50, 75); diff --git a/src/test/java/com/facebook/LinkBench/GeomDistTest.java b/src/test/java/com/facebook/LinkBench/GeomDistTest.java index b1e941e1..3b09995e 100644 --- a/src/test/java/com/facebook/LinkBench/GeomDistTest.java +++ b/src/test/java/com/facebook/LinkBench/GeomDistTest.java @@ -49,7 +49,7 @@ public void testGeom() { assertEquals(0.917646, d.cdf(7), 0.001); assertEquals(0.971752, d.cdf(10), 0.001); assertEquals(0.995252, d.cdf(15), 0.001); - + assertEquals(0.3, d.pdf(1), 0.001); assertEquals(0.21, d.pdf(2), 0.001); assertEquals(0.147, d.pdf(3), 0.001); diff --git a/src/test/java/com/facebook/LinkBench/GraphStoreTestBase.java b/src/test/java/com/facebook/LinkBench/GraphStoreTestBase.java index ad4f17bc..506e8201 100644 --- a/src/test/java/com/facebook/LinkBench/GraphStoreTestBase.java +++ b/src/test/java/com/facebook/LinkBench/GraphStoreTestBase.java @@ -43,7 +43,7 @@ public abstract class GraphStoreTestBase extends TestCase { */ protected abstract void initStore(Properties props) throws IOException, Exception; - + /** * Override to vary size of test * @return number of ids to use in testing @@ -51,34 +51,34 @@ protected abstract void initStore(Properties props) protected long getIDCount() { return 50000; } - + /** * Override to vary number of requests in test */ protected int getRequestCount() { return 100000; } - + /** * Override to vary maximum number of threads */ protected int maxConcurrentThreads() { return Integer.MAX_VALUE; } - + /** Get a new handle to the initialized store, wrapped in * DummyLinkStore * @return new handle to linkstore */ - protected abstract DummyLinkStore getStoreHandle(boolean initialized) + protected abstract DummyLinkStore getStoreHandle(boolean initialized) throws IOException, Exception; - + @Override protected void setUp() throws Exception { super.setUp(); initStore(basicProps()); } - + /** * Provide properties for basic test store * @return @@ -88,17 +88,17 @@ protected Properties basicProps() { props.setProperty(Config.DBID, testDB); return props; } - - + + public static void fillLoadProps(Properties props, long startId, long idCount, int linksPerId) { LinkStoreTestBase.fillLoadProps(props, startId, idCount, linksPerId); props.setProperty(Config.NODE_DATASIZE, "512.0"); props.setProperty(Config.NODE_ADD_DATAGEN, UniformDataGenerator.class.getName()); - props.setProperty(Config.NODE_ADD_DATAGEN_PREFIX + + props.setProperty(Config.NODE_ADD_DATAGEN_PREFIX + Config.UNIFORM_GEN_STARTBYTE, "0"); - props.setProperty(Config.NODE_ADD_DATAGEN_PREFIX + + props.setProperty(Config.NODE_ADD_DATAGEN_PREFIX + Config.UNIFORM_GEN_ENDBYTE, "255"); } @@ -115,8 +115,8 @@ public static void fillReqProps(Properties props, long startId, long idCount, props.setProperty(Config.PR_UPDATE_NODE, Double.toString(p_updatenode)); props.setProperty(Config.PR_DELETE_NODE, Double.toString(p_deletenode)); props.setProperty(Config.PR_GET_NODE, Double.toString(p_getnode)); - - props.setProperty(Config.NODE_READ_CONFIG_PREFIX + + + props.setProperty(Config.NODE_READ_CONFIG_PREFIX + Config.ACCESS_FUNCTION_SUFFIX, UniformDistribution.class.getName()); props.setProperty(Config.NODE_UPDATE_CONFIG_PREFIX + Config.ACCESS_FUNCTION_SUFFIX, AccessDistMode.ROUND_ROBIN.name()); @@ -124,107 +124,107 @@ public static void fillReqProps(Properties props, long startId, long idCount, Config.ACCESS_CONFIG_SUFFIX, "0"); props.setProperty(Config.NODE_DELETE_CONFIG_PREFIX + Config.ACCESS_FUNCTION_SUFFIX, UniformDistribution.class.getName()); - + props.setProperty(Config.NODE_DATASIZE, "1024"); props.setProperty(Config.NODE_ADD_DATAGEN, UniformDataGenerator.class.getName()); - props.setProperty(Config.NODE_ADD_DATAGEN_PREFIX + + props.setProperty(Config.NODE_ADD_DATAGEN_PREFIX + Config.UNIFORM_GEN_STARTBYTE, "0"); - props.setProperty(Config.NODE_ADD_DATAGEN_PREFIX + + props.setProperty(Config.NODE_ADD_DATAGEN_PREFIX + Config.UNIFORM_GEN_ENDBYTE, "255"); - + props.setProperty(Config.NODE_DATASIZE, "1024"); props.setProperty(Config.NODE_UP_DATAGEN, UniformDataGenerator.class.getName()); - props.setProperty(Config.NODE_UP_DATAGEN_PREFIX + + props.setProperty(Config.NODE_UP_DATAGEN_PREFIX + Config.UNIFORM_GEN_STARTBYTE, "0"); - props.setProperty(Config.NODE_UP_DATAGEN_PREFIX + + props.setProperty(Config.NODE_UP_DATAGEN_PREFIX + Config.UNIFORM_GEN_ENDBYTE, "255"); } - - + + /** * Test the full workload with node and link ops to exercise the * requester - * @throws Exception - * @throws IOException + * @throws Exception + * @throws IOException */ @Test public void testFullWorkload() throws IOException, Exception { long startId = 532; long idCount = getIDCount(); int linksPerId = 5; - + int requests = getRequestCount(); long timeLimit = requests; Properties props = basicProps(); fillLoadProps(props, startId, idCount, linksPerId); - - double p_add = 0.1, p_del = 0.05, p_up = 0.05, p_count = 0.05, + + double p_add = 0.1, p_del = 0.05, p_up = 0.05, p_count = 0.05, p_multiget = 0.05, p_getlinks = 0.1, - p_add_node = 0.2, p_up_node = 0.05, + p_add_node = 0.2, p_up_node = 0.05, p_del_node = 0.05, p_get_node = 0.3; fillReqProps(props, startId, idCount, requests, timeLimit, p_add * 100, p_del * 100, p_up * 100, p_count * 100, p_multiget * 100, p_getlinks * 100, p_add_node * 100, p_up_node * 100, p_del_node * 100, p_get_node * 100); - + try { Random rng = LinkStoreTestBase.createRNG(); - + LinkStoreTestBase.serialLoad(rng, logger, props, getStoreHandle(false)); serialLoadNodes(rng, logger, props, getStoreHandle(false)); - + DummyLinkStore reqStore = getStoreHandle(false); LatencyStats latencyStats = new LatencyStats(1); RequestProgress tracker = new RequestProgress(logger, requests, timeLimit, 1, 10000); - + // Test both link and node requests LinkBenchRequest requester = new LinkBenchRequest(reqStore, reqStore, props, latencyStats, System.out, tracker, rng, 0, 1); tracker.startTimer(); - + requester.run(); - + latencyStats.displayLatencyStats(); latencyStats.printCSVStats(System.out, true); - + assertEquals(requests, reqStore.adds + reqStore.updates + reqStore.deletes + reqStore.countLinks + reqStore.multigetLinks + reqStore.getLinkLists + reqStore.addNodes + reqStore.updateNodes + reqStore.deleteNodes + reqStore.getNodes); // Check that the proportion of operations is roughly right - within 1% // For now, updates are actually implemented as add operations - assertTrue(Math.abs(reqStore.adds / (double)requests - + assertTrue(Math.abs(reqStore.adds / (double)requests - (p_add + p_up)) < 0.01); - assertTrue(Math.abs(reqStore.updates / + assertTrue(Math.abs(reqStore.updates / (double)requests - 0.0) < 0.01); assertTrue(Math.abs(reqStore.deletes / (double)requests - p_del) < 0.01); - assertTrue(Math.abs(reqStore.countLinks / + assertTrue(Math.abs(reqStore.countLinks / (double)requests - p_count) < 0.01); assertTrue(Math.abs(reqStore.multigetLinks / (double)requests - p_multiget) < 0.01); - assertTrue(Math.abs(reqStore.getLinkLists / + assertTrue(Math.abs(reqStore.getLinkLists / (double)requests - p_getlinks) < 0.01); - assertTrue(Math.abs(reqStore.addNodes / + assertTrue(Math.abs(reqStore.addNodes / (double)requests - p_add_node) < 0.01); - assertTrue(Math.abs(reqStore.updateNodes / + assertTrue(Math.abs(reqStore.updateNodes / (double)requests - p_up_node) < 0.01); assertTrue(Math.abs(reqStore.deleteNodes / (double)requests - p_del_node) < 0.01); assertTrue(Math.abs(reqStore.getNodes / (double)requests - p_get_node) < 0.01); - + assertEquals(0, reqStore.bulkLoadCountOps); assertEquals(0, reqStore.bulkLoadLinkOps); } finally { try { LinkStoreTestBase.deleteIDRange(testDB, getStoreHandle(true), startId, idCount); - deleteNodeIDRange(testDB, LinkStore.DEFAULT_NODE_TYPE, + deleteNodeIDRange(testDB, LinkStore.DEFAULT_NODE_TYPE, getStoreHandle(true), startId, idCount); } catch (Throwable t) { System.err.println("Error during cleanup:"); @@ -247,12 +247,12 @@ static void deleteNodeIDRange(String testDB, int type, private void serialLoadNodes(Random rng, Logger logger, Properties props, DummyLinkStore storeHandle) throws Exception { storeHandle.initialize(props, Phase.LOAD, 0); - storeHandle.resetNodeStore(testDB, ConfigUtil.getLong(props, + storeHandle.resetNodeStore(testDB, ConfigUtil.getLong(props, Config.MIN_ID)); storeHandle.close(); // Close before passing to loader - + LatencyStats stats = new LatencyStats(1); - NodeLoader loader = new NodeLoader(props, logger, storeHandle, rng, + NodeLoader loader = new NodeLoader(props, logger, storeHandle, rng, stats, System.out, 0); loader.run(); } diff --git a/src/test/java/com/facebook/LinkBench/HarmonicTest.java b/src/test/java/com/facebook/LinkBench/HarmonicTest.java index 1b4c9719..ecbda2fe 100644 --- a/src/test/java/com/facebook/LinkBench/HarmonicTest.java +++ b/src/test/java/com/facebook/LinkBench/HarmonicTest.java @@ -33,14 +33,14 @@ public void testHarmonic() { assertEquals(2679914.0, Harmonic.generalizedHarmonic(12345678, 0.1), 1); } - + @Test public void testApprox() { - - // Test that approximation is close to actual for a range of shapes and + + // Test that approximation is close to actual for a range of shapes and // ns double shapes[] = {0.01, 0.1, 0.5, 0.9, 0.99}; - + for (long i = 0; i < 30; i+=4) { long n = (long)Math.pow(2, i); for (double shape: shapes) { diff --git a/src/test/java/com/facebook/LinkBench/ID2ChooserTest.java b/src/test/java/com/facebook/LinkBench/ID2ChooserTest.java index b1d04d0a..5617882f 100644 --- a/src/test/java/com/facebook/LinkBench/ID2ChooserTest.java +++ b/src/test/java/com/facebook/LinkBench/ID2ChooserTest.java @@ -29,7 +29,7 @@ public class ID2ChooserTest extends TestCase { Properties props; - + @Override public void setUp() { props = new Properties(); @@ -38,7 +38,7 @@ public void setUp() { props.setProperty(Config.NLINKS_PREFIX + "shape", "0.5"); props.setProperty(Config.NLINKS_PREFIX + "scale", "1000000"); } - + @Test public void testNoLoadCollisions() { long n = 10000; @@ -46,25 +46,25 @@ public void testNoLoadCollisions() { long max = n + min; long seed = 5313242; Random rng = new Random(seed); - + ID2Chooser c = new ID2Chooser(props, min, max, 1, 1); // Check we don't get the same id2 more than once (i.e. duplicate links) - + int nlinks = 50; HashMap seen = new HashMap(); - + long id1 = 1234; for (int i = 0; i < nlinks; i++) { long id2 = c.chooseForLoad(rng, id1, LinkStore.DEFAULT_LINK_TYPE, i); Integer j = seen.get(id2); if (j != null) { fail("Same link generated twice: (" + id1 + ", " + id2 + ") for " + - " indices " + j + " and " + i); + " indices " + j + " and " + i); } seen.put(id2, i); } } - + @Test public void testChooseForOp() { // Currently just exercise the code: I don't have a good way to verify the @@ -79,12 +79,12 @@ public void testChooseForOp() { for (int i = 0; i < trials; i++) { long id2 = c.chooseForOp(rng, i + min, LinkStore.DEFAULT_LINK_TYPE, 1.0); assert(id2 >= min); - + id2 = c.chooseForOp(rng, i + min, LinkStore.DEFAULT_LINK_TYPE, 0.5); assert(id2 >= min); } } - + /** * Check that the choosing mechanism is generating id2s with the * right probability of a loaded link matching @@ -93,7 +93,7 @@ public void testChooseForOp() { public void testMatchPercent() { long seed = 15325435L; Random rng = new Random(seed); - + int minid = 500, maxid=1000000; ID2Chooser chooser = new ID2Chooser(props, minid, maxid, 1, 0); for (int id1 = minid; id1 < maxid; id1 += 3763) { @@ -104,31 +104,31 @@ public void testMatchPercent() { LinkStore.DEFAULT_LINK_TYPE, i); existing.add(id2); } - + int trials = 10000; - + int hit = 0; // hit for prob = 50% - + for (int i = 0; i < trials; i++) { // Test with 100% prob of hit - long id2 = chooser.chooseForOp(rng, id1, + long id2 = chooser.chooseForOp(rng, id1, LinkStore.DEFAULT_LINK_TYPE, 1.0); assertTrue(existing.contains(id2) || existing.size() == 0); - + // Test with 50% prob of hit id2 = chooser.chooseForOp(rng, id1, LinkStore.DEFAULT_LINK_TYPE, 0.5); if (existing.contains(id2)) { hit++; } } - + double hitPercent = hit / (double)trials; if (existing.size() > 0 && Math.abs(0.5 - hitPercent) > 0.05) { fail(hitPercent * 100 + "% of ids2 were hits for id1 " + id1); } } } - + /** * Check that link counts work for multiple link types */ @@ -138,16 +138,16 @@ public void testLinkCount() { Properties newProps = new Properties(props); int nLinkTypes = 10; newProps.setProperty(Config.LINK_TYPE_COUNT, Integer.toString(nLinkTypes)); - + ID2Chooser chooser = new ID2Chooser(newProps, startid, maxid, 1, 0); long linkTypes[] = chooser.getLinkTypes(); assertEquals(nLinkTypes, linkTypes.length); - + // Check it works for some different IDs for (long id = startid; id < maxid; id += 7) { long totalCount = 0; for (long linkType: linkTypes) { - totalCount += chooser.calcLinkCount(id, linkType); + totalCount += chooser.calcLinkCount(id, linkType); } assertEquals(chooser.calcTotalLinkCount(id), totalCount); } diff --git a/src/test/java/com/facebook/LinkBench/InvertibleShufflerTest.java b/src/test/java/com/facebook/LinkBench/InvertibleShufflerTest.java index cf95a826..64580097 100644 --- a/src/test/java/com/facebook/LinkBench/InvertibleShufflerTest.java +++ b/src/test/java/com/facebook/LinkBench/InvertibleShufflerTest.java @@ -20,7 +20,7 @@ import org.junit.Test; import junit.framework.TestCase; -/** +/** * Check that the shuffler obeys expected invariants */ public class InvertibleShufflerTest extends TestCase { @@ -38,18 +38,18 @@ public void testShuffleSmallRange() { testShuffle(0, 10, true, seed, 15); testShuffle(0, 100, true, seed, 11); } - + @Test public void testShuffleMedRange() { testShuffle(512, 10543, false, 13, 7); testShuffle(512, 10543, false, 13, 7, 27, 140); } - + @Test public void testShuffleLargeRange() { testShuffle(12345, 123456, false, 13, 7, 27, 140); } - + /** * Check that result is a valid permutation (i.e. a 1->1 mapping) * @param minId @@ -61,22 +61,22 @@ public static void testShuffle(int minId, int maxId, boolean print, String shuffleDesc = String.format( "Permuting range [%d,%d) with params %s", minId, maxId, Arrays.toString(params)); - + if (print) { System.err.println(shuffleDesc); } - - + + int n = maxId - minId; - + //ProbDistShuffler shuf = new ProbDistShuffler(params[0], (int)params[1], n); InvertibleShuffler shuf = new InvertibleShuffler(params[0], (int)params[1], n); - + long reverse[] = new long[n]; // Store the reverse permutation // Store if ID has appeared (inited to false) boolean exists[] = new boolean[n]; - + for (int i = minId; i < maxId; i++) { //long lj = Shuffler.getPermutationValue(i, minId, maxId, params); long lj = minId + shuf.permute(i - minId); @@ -87,11 +87,11 @@ public static void testShuffle(int minId, int maxId, boolean print, } assertTrue(lj >= minId); assertTrue(lj < maxId); - int j = (int)lj; // Must be in integer range + int j = (int)lj; // Must be in integer range if (exists[j - minId]) { fail(String.format( - "Error with test %s: collision. p(%d) = p(%d) = %d", - shuffleDesc, i, reverse[j - minId], j)); + "Error with test %s: collision. p(%d) = p(%d) = %d", + shuffleDesc, i, reverse[j - minId], j)); } reverse[j - minId] = i; exists[j - minId] = true; diff --git a/src/test/java/com/facebook/LinkBench/LinkStoreTestBase.java b/src/test/java/com/facebook/LinkBench/LinkStoreTestBase.java index 6339b3c0..1dcdf46b 100644 --- a/src/test/java/com/facebook/LinkBench/LinkStoreTestBase.java +++ b/src/test/java/com/facebook/LinkBench/LinkStoreTestBase.java @@ -39,15 +39,15 @@ import com.facebook.LinkBench.stats.LatencyStats; /** - * This test implements unit tests that *all* implementations of LinkStore + * This test implements unit tests that *all* implementations of LinkStore * should pass. - * + * * Different implementations of LinkStore will require different configuration - * and different setups for testing, so in order to test out a particular + * and different setups for testing, so in order to test out a particular * LinkStore implementation, you can subclass this test and implement the * required abstract methods so that the test store is initialized correctly * and all required configuration properties are filled in. - * + * * @author tarmstrong */ public abstract class LinkStoreTestBase extends TestCase { @@ -63,7 +63,7 @@ public abstract class LinkStoreTestBase extends TestCase { */ protected abstract void initStore(Properties props) throws IOException, Exception; - + /** * Override to vary size of test * @return number of ids to use in testing @@ -71,34 +71,34 @@ protected abstract void initStore(Properties props) protected long getIDCount() { return 50000; } - + /** * Override to vary number of requests in test */ protected int getRequestCount() { return 100000; } - + /** * Override to vary maximum number of threads */ protected int maxConcurrentThreads() { return Integer.MAX_VALUE; } - + /** Get a new handle to the initialized store, wrapped in * DummyLinkStore * @return new handle to linkstore */ - protected abstract DummyLinkStore getStoreHandle(boolean initialized) + protected abstract DummyLinkStore getStoreHandle(boolean initialized) throws IOException, Exception; - + @Override protected void setUp() throws Exception { super.setUp(); initStore(basicProps()); } - + /** * Provide properties for basic test store * @return @@ -108,26 +108,26 @@ protected Properties basicProps() { props.setProperty(Config.DBID, testDB); return props; } - + public static void fillLoadProps(Properties props, long startId, long idCount, int linksPerId) { props.setProperty(Config.MIN_ID,Long.toString(startId)); props.setProperty(Config.MAX_ID, Long.toString(startId + idCount)); props.setProperty(Config.RANDOM_ID2_MAX, "0"); // Fixed number of rows - props.setProperty(Config.NLINKS_FUNC, LinkDistMode.CONST.name()); + props.setProperty(Config.NLINKS_FUNC, LinkDistMode.CONST.name()); props.setProperty(Config.NLINKS_CONFIG, "0"); // ignored props.setProperty(Config.NLINKS_DEFAULT, Integer.toString(linksPerId)); props.setProperty(Config.DISPLAY_FREQ, "10"); // Show stats frequently props.setProperty(Config.MAX_STAT_SAMPLES, "10000"); - + props.setProperty(Config.LINK_DATASIZE, "100.0"); props.setProperty(Config.LINK_ADD_DATAGEN, UniformDataGenerator.class.getName()); - props.setProperty(Config.LINK_ADD_DATAGEN_PREFIX + + props.setProperty(Config.LINK_ADD_DATAGEN_PREFIX + Config.UNIFORM_GEN_STARTBYTE, "0"); - props.setProperty(Config.LINK_ADD_DATAGEN_PREFIX + + props.setProperty(Config.LINK_ADD_DATAGEN_PREFIX + Config.UNIFORM_GEN_ENDBYTE, "255"); } @@ -139,17 +139,17 @@ public static void fillReqProps(Properties props, long startId, long idCount, props.setProperty(Config.MAX_ID, Long.toString(startId + idCount)); props.setProperty(Config.NUM_REQUESTS, Long.toString(requests)); props.setProperty(Config.MAX_TIME, Long.toString(timeLimit)); - + props.setProperty(Config.RANDOM_ID2_MAX, "0"); props.setProperty(Config.ID2GEN_CONFIG, "0"); - + props.setProperty(Config.PR_ADD_LINK, Double.toString(p_addlink)); props.setProperty(Config.PR_DELETE_LINK, Double.toString(p_deletelink)); props.setProperty(Config.PR_UPDATE_LINK, Double.toString(p_updatelink)); props.setProperty(Config.PR_COUNT_LINKS, Double.toString(p_countlink)); props.setProperty(Config.PR_GET_LINK, Double.toString(p_getlink)); props.setProperty(Config.PR_GET_LINK_LIST, Double.toString(p_getlinklist)); - + props.setProperty(Config.WRITE_FUNCTION, UniformDistribution.class.getName()); props.setProperty(Config.READ_FUNCTION, AccessDistMode.RECIPROCAL.name()); @@ -159,34 +159,34 @@ public static void fillReqProps(Properties props, long startId, long idCount, props.setProperty(Config.READ_UNCORR_FUNCTION, ZipfDistribution.class.getName()); props.setProperty(Config.READ_UNCORR_CONFIG_PREFIX + "shape", "0.5"); - - + + if (enableMultiget) { - props.setProperty(Config.LINK_MULTIGET_DIST, + props.setProperty(Config.LINK_MULTIGET_DIST, GeometricDistribution.class.getName()); props.setProperty(Config.LINK_MULTIGET_DIST_MIN, "0"); props.setProperty(Config.LINK_MULTIGET_DIST_MAX, "10"); props.setProperty(Config.LINK_MULTIGET_DIST_PREFIX + GeometricDistribution.PROB_PARAM_KEY, "0.8"); } - + props.setProperty(Config.LINK_DATASIZE, "200"); props.setProperty(Config.LINK_ADD_DATAGEN, UniformDataGenerator.class.getName()); - props.setProperty(Config.LINK_ADD_DATAGEN_PREFIX + + props.setProperty(Config.LINK_ADD_DATAGEN_PREFIX + Config.UNIFORM_GEN_STARTBYTE, "0"); - props.setProperty(Config.LINK_ADD_DATAGEN_PREFIX + + props.setProperty(Config.LINK_ADD_DATAGEN_PREFIX + Config.UNIFORM_GEN_ENDBYTE, "255"); - + props.setProperty(Config.LINK_UP_DATAGEN, UniformDataGenerator.class.getName()); - props.setProperty(Config.LINK_UP_DATAGEN_PREFIX + + props.setProperty(Config.LINK_UP_DATAGEN_PREFIX + Config.UNIFORM_GEN_STARTBYTE, "0"); - props.setProperty(Config.LINK_UP_DATAGEN_PREFIX + + props.setProperty(Config.LINK_UP_DATAGEN_PREFIX + Config.UNIFORM_GEN_ENDBYTE, "255"); } - /** + /** * Utility to create a random number generator and print * the seed for later reproducibility of test failures * @return @@ -204,7 +204,7 @@ public void testOneLink() throws IOException, Exception { DummyLinkStore store = getStoreHandle(true); long id1 = 1123, id2 = 1124, ltype = 321; - Link writtenLink = new Link(id1, ltype, id2, + Link writtenLink = new Link(id1, ltype, id2, LinkStore.VISIBILITY_DEFAULT, new byte[] {0x1}, 1, 1994); store.addLink(testDB, writtenLink, true); if (store.isRealLinkStore()) { @@ -216,13 +216,13 @@ public void testOneLink() throws IOException, Exception { } assertEquals(1, store.countLinks(testDB, id1, ltype)); } - + // Try expunge store.deleteLink(testDB, id1, ltype, id2, true, true); assertNull(store.getLink(testDB, id1, ltype, id2)); assertNull(store.getLinkList(testDB, id1, ltype)); assertEquals(0, store.countLinks(testDB, id1, ltype)); - + store.addLink(testDB, writtenLink, true); if (store.isRealLinkStore()) { assertNotNull(store.getLink(testDB, id1, ltype, id2)); @@ -241,7 +241,7 @@ public void testOneLink() throws IOException, Exception { assertEquals(0, store.countLinks(testDB, id1, ltype)); assertNull(store.getLinkList(testDB, id1, ltype)); } - + // Update link: check it is unhidden store.updateLink(testDB, writtenLink, true); if (store.isRealLinkStore()) { @@ -251,7 +251,7 @@ public void testOneLink() throws IOException, Exception { assertEquals(1, links.length); assertTrue(writtenLink.equals(links[0])); } - + // Update link but don't change, check nothing changes store.updateLink(testDB, writtenLink, true); if (store.isRealLinkStore()) { @@ -261,16 +261,16 @@ public void testOneLink() throws IOException, Exception { assertEquals(1, links.length); assertTrue(writtenLink.equals(links[0])); } - + store.deleteLink(testDB, id1, ltype, id2, true, true); } - + @Test public void testMultipleLinks() throws Exception, IOException { DummyLinkStore store = getStoreHandle(true); long ida = 5434, idb = 5435, idc = 9999, idd = 9998; long ltypea = 1, ltypeb = 2; - + byte data[] = new byte[] {0xf, 0xa, 0xc, 0xe, 0xb, 0x0, 0x0, 0xc}; long t = 10000000; Link links[] = new Link[] { @@ -296,42 +296,42 @@ public void testMultipleLinks() throws Exception, IOException { assertEquals(1, store.countLinks(testDB, ida, ltypeb)); assertEquals(0, store.countLinks(testDB, idb, ltypea)); assertEquals(3, store.countLinks(testDB, idb, ltypeb)); - + Link retrieved[]; - + retrieved = store.getLinkList(testDB, ida, ltypea); assertEquals(1, retrieved.length); assertTrue(links[0].equals(retrieved[0])); - + retrieved = store.getLinkList(testDB, ida, ltypeb); assertEquals(1, retrieved.length); assertTrue(links[1].equals(retrieved[0])); - + retrieved = store.getLinkList(testDB, idb, ltypeb); // Check link list, Four matching links, one hidden checkExpectedList(store, idb, ltypeb, links[5], links[2], links[3]); - + // Check limit - retrieved = store.getLinkList(testDB, idb, ltypeb, + retrieved = store.getLinkList(testDB, idb, ltypeb, 0, t + 100, 0, 1); assertEquals(1, retrieved.length); assertTrue(links[5].equals(retrieved[0])); - + //Check offset + limit - retrieved = store.getLinkList(testDB, idb, ltypeb, + retrieved = store.getLinkList(testDB, idb, ltypeb, 0, t + 100, 1, 2); assertEquals(2, retrieved.length); assertTrue(links[2].equals(retrieved[0])); assertTrue(links[3].equals(retrieved[1])); - + // Check range filtering - retrieved = store.getLinkList(testDB, idb, ltypeb, + retrieved = store.getLinkList(testDB, idb, ltypeb, t + 1, t + 2, 0, Integer.MAX_VALUE); assertEquals(1, retrieved.length); assertTrue(links[2].equals(retrieved[0])); } } - + /** * Simple test to make sure multiget works * @throws IOException @@ -349,7 +349,7 @@ public void testMultiget() throws IOException, Exception { store.addLink(testDB, a, true); store.addLink(testDB, b, true); // Retrieve the two added links - Link l[] = store.multigetLinks(testDB, a.id1, a.link_type, + Link l[] = store.multigetLinks(testDB, a.id1, a.link_type, new long[] {a.id2, b.id2, 1234}); if (store.isRealLinkStore()) { assertEquals(2, l.length); @@ -362,7 +362,7 @@ public void testMultiget() throws IOException, Exception { } } } - + /** * Regression test for flaw in MySql where visibility is assumed to * be default on add @@ -375,25 +375,25 @@ public void testHiding() throws Exception { System.currentTimeMillis()); store.addLink(testDB, l, true); checkExpectedList(store, 1, 1, new Link[0]); - + // Check that updating works right store.deleteLink(testDB, 1, 1, 1, true, false); checkExpectedList(store, 1, 1, new Link[0]); - + // Make it visible l.visibility = LinkStore.VISIBILITY_DEFAULT; store.addLink(testDB, l, true); checkExpectedList(store, 1, 1, l); - + // Expunge store.deleteLink(testDB, 1, 1, 1, true, true); checkExpectedList(store, 1, 1, new Link[0]); } - + /** * Test that all fields are updated correctly on update - * @throws Exception - * @throws IOException + * @throws Exception + * @throws IOException */ @Test public void testOverwrite() throws IOException, Exception { @@ -401,20 +401,20 @@ public void testOverwrite() throws IOException, Exception { Link orig = new Link(id1, 1, 1, LinkStore.VISIBILITY_DEFAULT, new byte[] {'1','1','1'}, 0, 1); Link changed = orig.clone(); - changed.data = new byte[] {'2', '2', '2'}; + changed.data = new byte[] {'2', '2', '2'}; changed.version = 1; changed.time = 2; DummyLinkStore store = getStoreHandle(true); - + store.addLink(testDB, orig, true); - + // Check added ok Link tmp = store.getLink(testDB, orig.id1, orig.link_type, orig.id2); if (store.isRealLinkStore()) { assertTrue(orig.equals(tmp)); assertEquals(1, store.countLinks(testDB, orig.id1, orig.link_type)); } - + // Overwrite, then check update worked for all fields store.addLink(testDB, changed, true); tmp = store.getLink(testDB, orig.id1, orig.link_type, orig.id2); @@ -422,7 +422,7 @@ public void testOverwrite() throws IOException, Exception { assertTrue(changed.equals(tmp)); assertEquals(1, store.countLinks(testDB, orig.id1, orig.link_type)); } - + // Add hidden link, check update happened Link hidden = orig.clone(); hidden.visibility = LinkStore.VISIBILITY_HIDDEN; @@ -433,16 +433,16 @@ public void testOverwrite() throws IOException, Exception { assertEquals(0, store.countLinks(testDB, orig.id1, orig.link_type)); } } - + /** * Regression test for bad handling of string escaping */ @Test public void testSqlInjection() throws IOException, Exception { - Link l = new Link(1, 1, 1, LinkStore.VISIBILITY_DEFAULT, + Link l = new Link(1, 1, 1, LinkStore.VISIBILITY_DEFAULT, "' asdfasdf".getBytes(), 1, 1); byte updateData[] = "';\\".getBytes(); - + testAddThenUpdate(l, updateData); } @@ -450,13 +450,13 @@ private void testAddThenUpdate(Link l, byte[] updateData) throws IOException, Exception { DummyLinkStore ls = getStoreHandle(true); ls.addLink(testDB, l, true); - + Link l2 = ls.getLink(testDB, 1, 1, 1); if (ls.isRealLinkStore()) { assertNotNull(l2); assertTrue(l.equals(l2)); } - + l.data = updateData; ls.updateLink(testDB, l, true); l2 = ls.getLink(testDB, 1, 1, 1); @@ -465,20 +465,20 @@ private void testAddThenUpdate(Link l, byte[] updateData) throws IOException, assertTrue(l.equals(l2)); } } - + /** Check handling of bytes 0-127 */ @Test public void testBinary1() throws IOException, Exception { binaryDataTest(0, 128); } - + /** Check handling of bytes 160-256 */ @Test public void testBinary2() throws IOException, Exception { int start = 160; binaryDataTest(start, 256-start); } - + /** Check handling of bytes 128-159 */ @Test public void testBinary3() throws IOException, Exception { @@ -499,7 +499,7 @@ private void binaryDataTest(int startByte, int dataMaxSize) byte b = (byte)((i + startByte) % 256); data[i] = b; } - Link l = new Link(1, 1, 1, LinkStore.VISIBILITY_DEFAULT, + Link l = new Link(1, 1, 1, LinkStore.VISIBILITY_DEFAULT, data, 1, 1); // Different length and data byte updateData[] = new byte[dataMaxSize/2]; @@ -508,14 +508,14 @@ private void binaryDataTest(int startByte, int dataMaxSize) } testAddThenUpdate(l, updateData); } - - - + + + /** * Generic test for a loader using a wrapped LinkStore * implementation - * @throws Exception - * @throws IOException + * @throws Exception + * @throws IOException */ @Test public void testLoader() throws IOException, Exception { @@ -525,25 +525,25 @@ public void testLoader() throws IOException, Exception { Properties props = basicProps(); fillLoadProps(props, startId, idCount, linksPerId); - + initStore(props); DummyLinkStore store = getStoreHandle(false); - + try { Random rng = createRNG(); - + serialLoad(rng, logger, props, store); - + long testEndTime = System.currentTimeMillis(); - + assertFalse(store.initialized); // Check was closed - + /* Validate results */ if (store.bulkLoadBatchSize() > 0) { assertEquals(idCount, store.bulkLoadCountRows); } assertEquals(idCount * linksPerId, store.bulkLoadLinkRows + store.adds); - + if (store.isRealLinkStore()) { // old store was closed by loader store.initialize(props, Phase.REQUEST, 0); @@ -560,41 +560,41 @@ public void testLoader() throws IOException, Exception { } /** - * Run the requester against + * Run the requester against * This test validates both the requester (by looking at counts to make * sure it at least did the right number of ops) and the LinkStore * (by stress-testing it). - * @throws Exception - * @throws IOException + * @throws Exception + * @throws IOException */ @Test public void testRequester() throws IOException, Exception { long startId = 532; long idCount = getIDCount(); int linksPerId = 5; - + int requests = getRequestCount(); long timeLimit = requests; Properties props = basicProps(); fillLoadProps(props, startId, idCount, linksPerId); - - double p_add = 0.2, p_del = 0.2, p_up = 0.1, p_count = 0.1, + + double p_add = 0.2, p_del = 0.2, p_up = 0.1, p_count = 0.1, p_multiget = 0.2, p_getlinks = 0.2; fillReqProps(props, startId, idCount, requests, timeLimit, p_add * 100, p_del * 100, p_up * 100, p_count * 100, p_multiget * 100, p_getlinks * 100, true); - + try { Random rng = createRNG(); - + serialLoad(rng, logger, props, getStoreHandle(false)); - + DummyLinkStore reqStore = getStoreHandle(false); LatencyStats latencyStats = new LatencyStats(1); RequestProgress tracker = new RequestProgress(logger, requests, timeLimit, 0, 1000); - + LinkBenchRequest requester = new LinkBenchRequest(reqStore, null, props, latencyStats, System.out, tracker, rng, 0, 1); @@ -602,22 +602,22 @@ public void testRequester() throws IOException, Exception { requester.run(); latencyStats.displayLatencyStats(); latencyStats.printCSVStats(System.out, true); - + assertEquals(requests, reqStore.adds + reqStore.updates + reqStore.deletes + reqStore.countLinks + reqStore.multigetLinks + reqStore.getLinkLists); // Check that the proportion of operations is roughly right - within 1% // For now, updates are actually implemented as add operations - assertTrue(Math.abs(reqStore.adds / (double)requests - + assertTrue(Math.abs(reqStore.adds / (double)requests - (p_add + p_up)) < 0.01); - assertTrue(Math.abs(reqStore.updates / + assertTrue(Math.abs(reqStore.updates / (double)requests - 0.0) < 0.01); assertTrue(Math.abs(reqStore.deletes / (double)requests - p_del) < 0.01); - assertTrue(Math.abs(reqStore.countLinks / + assertTrue(Math.abs(reqStore.countLinks / (double)requests - p_count) < 0.01); assertTrue(Math.abs(reqStore.multigetLinks / (double)requests - p_multiget) < 0.01); - assertTrue(Math.abs(reqStore.getLinkLists / + assertTrue(Math.abs(reqStore.getLinkLists / (double)requests - p_getlinks) < 0.01); assertEquals(0, reqStore.bulkLoadCountOps); assertEquals(0, reqStore.bulkLoadLinkOps); @@ -626,11 +626,11 @@ public void testRequester() throws IOException, Exception { } System.err.println("Done!"); } - + /** * Test that the requester throttling slows down requests - * @throws Exception - * @throws IOException + * @throws Exception + * @throws IOException */ @Test public void testRequesterThrottling() throws IOException, Exception { @@ -638,7 +638,7 @@ public void testRequesterThrottling() throws IOException, Exception { // Small test long idCount = getIDCount() / 10; int linksPerId = 3; - + Properties props = basicProps(); int requests = 2000; long timeLimit = requests; @@ -647,23 +647,23 @@ public void testRequesterThrottling() throws IOException, Exception { fillReqProps(props, startId, idCount, requests, timeLimit, 20, 20, 10, 10, 20, 20, false); props.setProperty("requestrate", Integer.toString(requestsPerSec)); - + try { Random rng = createRNG(); - + serialLoad(rng, logger, props, getStoreHandle(false)); RequestProgress tracker = new RequestProgress(logger, requests, timeLimit, 2, 1000); - + DummyLinkStore reqStore = getStoreHandle(false); LinkBenchRequest requester = new LinkBenchRequest(reqStore, null, props, new LatencyStats(1), System.out, tracker, rng, 0, 1); - + long startTime = System.currentTimeMillis(); tracker.startTimer(); requester.run(); long endTime = System.currentTimeMillis(); - + assertEquals(requests, reqStore.adds + reqStore.updates + reqStore.deletes + reqStore.countLinks + reqStore.multigetLinks + reqStore.getLinkLists); double actualArrivalRate = 1000 * requests / (double)(endTime - startTime); @@ -676,7 +676,7 @@ props, new LatencyStats(1), System.out, tracker, } System.err.println("Done!"); } - + /** * Check that the get link list history requests occur */ @@ -686,16 +686,16 @@ public void testHistoryRequests() throws Exception { // Few ids with many links long idCount = 10; int rangeLimit = 10; - + int linksPerId = (int) (rangeLimit * 20); - + Properties props = basicProps(); double pHistory = 0.25; // Quarter history requests - int requests = 50000; // enough requests that we should get 20%+ history + int requests = 50000; // enough requests that we should get 20%+ history // queries with something in cache. Many requests to // ensure we cycle through lists multiple times - long timeLimit = requests; - + long timeLimit = requests; + fillLoadProps(props, startId, idCount, linksPerId); fillReqProps(props, startId, idCount, requests, timeLimit, 0, 0, 0, 0, 0, 100, false); @@ -703,16 +703,16 @@ public void testHistoryRequests() throws Exception { props.setProperty(Config.READ_FUNCTION, UniformDistribution.class.getName()); // Test blending on reads props.setProperty(Config.READ_UNCORR_BLEND, "0.0"); - + props.setProperty(Config.PR_GETLINKLIST_HISTORY, Double.toString( pHistory * 100)); - + try { Random rng = createRNG(); - + serialLoad(rng, logger, props, getStoreHandle(false)); RequestProgress tracker = new RequestProgress(logger, requests, timeLimit, 0, 1000); - + DummyLinkStore reqStore = getStoreHandle(false); reqStore.setRangeLimit(rangeLimit); // Small limit for testing LatencyStats latencyStats = new LatencyStats(1); @@ -722,12 +722,12 @@ public void testHistoryRequests() throws Exception { tracker.startTimer(); requester.run(); latencyStats.displayLatencyStats(); - + assertEquals(requests, reqStore.getLinkLists); - - double actualPHistory = reqStore.getLinkListsHistory / + + double actualPHistory = reqStore.getLinkListsHistory / (double) reqStore.getLinkLists; - + System.err.println("# getLinkLists: " + reqStore.getLinkLists + " # getLinkLists for history: " + reqStore.getLinkListsHistory + " " + (actualPHistory * 100) + "%"); @@ -753,12 +753,12 @@ private void checkExpectedList(DummyLinkStore store, for (int i = 0; i < expected.length; i++) { if (!expected[i].equals(actual[i])) { fail("Mismatch between result lists. Expected: " + - Arrays.toString(expected) + " Actual: " + Arrays.toString(actual)); + Arrays.toString(expected) + " Actual: " + Arrays.toString(actual)); } } } } - + /** * Use the LinkBenchLoad class to do a serial load of data * @param logger @@ -771,30 +771,30 @@ private void checkExpectedList(DummyLinkStore store, static void serialLoad(Random rng, Logger logger, Properties props, DummyLinkStore store) throws IOException, Exception { LatencyStats latencyStats = new LatencyStats(1); - + /* Load up queue with work */ BlockingQueue chunk_q = new LinkedBlockingQueue(); long startId = ConfigUtil.getLong(props, Config.MIN_ID); long idCount = ConfigUtil.getLong(props, Config.MAX_ID) - startId; - + int chunkSize = 128; int seq = 0; for (long i = startId; i < startId + idCount; i+= chunkSize) { - LoadChunk chunk = new LoadChunk(seq, i, + LoadChunk chunk = new LoadChunk(seq, i, Math.min(idCount + startId, i + chunkSize), rng); chunk_q.add(chunk); seq++; } chunk_q.add(LoadChunk.SHUTDOWN); - - + + LoadProgress tracker = new LoadProgress(logger, idCount, 1000); tracker.startTimer(); - LinkBenchLoad loader = new LinkBenchLoad(store, + LinkBenchLoad loader = new LinkBenchLoad(store, props, latencyStats, System.out, 0, false, chunk_q, tracker); /* Run the loading process */ loader.run(); - + logger.info("Loaded " + (store.adds + store.bulkLoadLinkRows) + " links. " + store.adds + " individually " + " and " + store.bulkLoadLinkRows + " in rows"); @@ -806,7 +806,7 @@ private void validateLoadedData(Logger logger, DummyLinkStore wrappedStore, for (long i = startId; i < startId + idCount; i++) { assertEquals(wrappedStore.countLinks(testDB, i, LinkStore.DEFAULT_LINK_TYPE), linksPerId); - + Link links[] = wrappedStore.getLinkList(testDB, i, LinkStore.DEFAULT_LINK_TYPE); if (linksPerId == 0) { assertTrue(links == null); diff --git a/src/test/java/com/facebook/LinkBench/LogNormalTest.java b/src/test/java/com/facebook/LinkBench/LogNormalTest.java index d6a82b56..68c7b514 100644 --- a/src/test/java/com/facebook/LinkBench/LogNormalTest.java +++ b/src/test/java/com/facebook/LinkBench/LogNormalTest.java @@ -39,7 +39,7 @@ protected Properties getDistParams() { protected double tolerance() { return 0.05; } - + /** * Sanity check values */ @@ -49,8 +49,8 @@ public void testLogNormal() { d.init(0, 100, median, 1); // CDF of median should be 0.5 by def. assertEquals(0.5, d.cdf(median), 0.01); - - + + // Precomputed points d.init(0, 1000, 100, 1); assertEquals(0.033434, d.cdf(16), 0.0001); diff --git a/src/test/java/com/facebook/LinkBench/MemoryLinkStoreTest.java b/src/test/java/com/facebook/LinkBench/MemoryLinkStoreTest.java index ffa34c7d..cb15b044 100644 --- a/src/test/java/com/facebook/LinkBench/MemoryLinkStoreTest.java +++ b/src/test/java/com/facebook/LinkBench/MemoryLinkStoreTest.java @@ -19,21 +19,21 @@ import java.util.Properties; public class MemoryLinkStoreTest extends LinkStoreTestBase { - + MemoryLinkStore store; - + @Override public void setUp() throws Exception { super.setUp(); } - + @Override protected Properties basicProps() { Properties props = super.basicProps(); props.setProperty(Config.LINKSTORE_CLASS, MemoryLinkStore.class.getName()); return props; } - + @Override protected void initStore(Properties props) throws IOException, Exception { @@ -42,8 +42,8 @@ protected void initStore(Properties props) throws IOException, @Override protected DummyLinkStore getStoreHandle(boolean initialized) { - // Return a new memory link store handle. The underlying link store doesn't need to be - // initialized, so just set wrapper to correct init status + // Return a new memory link store handle. The underlying link store doesn't + // need to be initialized, so just set wrapper to correct init status return new DummyLinkStore(store.newHandle(), initialized); } diff --git a/src/test/java/com/facebook/LinkBench/MySqlGraphStoreTest.java b/src/test/java/com/facebook/LinkBench/MySqlGraphStoreTest.java index b02b4cbd..5a1ce2fc 100644 --- a/src/test/java/com/facebook/LinkBench/MySqlGraphStoreTest.java +++ b/src/test/java/com/facebook/LinkBench/MySqlGraphStoreTest.java @@ -23,7 +23,7 @@ public class MySqlGraphStoreTest extends GraphStoreTestBase { private Properties props; private Connection conn; - + @Override protected void initStore(Properties props) throws IOException, Exception { this.props = props; diff --git a/src/test/java/com/facebook/LinkBench/MySqlLinkStoreTest.java b/src/test/java/com/facebook/LinkBench/MySqlLinkStoreTest.java index 7e62680d..2254e17f 100644 --- a/src/test/java/com/facebook/LinkBench/MySqlLinkStoreTest.java +++ b/src/test/java/com/facebook/LinkBench/MySqlLinkStoreTest.java @@ -21,18 +21,18 @@ /** * Test the MySQL LinkStore implementation. - * + * * Assumes that the database specified by the testDB field has been created * with permissions for a user/pass linkbench/linkbench to create tables, select, * insert, delete, etc. */ public class MySqlLinkStoreTest extends LinkStoreTestBase { - + private Connection conn; - + /** Properties for last initStore call */ private Properties currProps; - + @Override protected long getIDCount() { // Make test smaller so that it doesn't take too long @@ -44,7 +44,7 @@ protected int getRequestCount() { // Fewer requests to keep test quick return 10000; } - + protected Properties basicProps() { Properties props = super.basicProps(); MySqlTestConfig.fillMySqlTestServerProps(props); @@ -63,7 +63,7 @@ protected void initStore(Properties props) throws IOException, Exception { MySqlTestConfig.createTestTables(conn, testDB); } - + @Override public DummyLinkStore getStoreHandle(boolean initialize) throws IOException, Exception { diff --git a/src/test/java/com/facebook/LinkBench/MySqlNodeStoreTest.java b/src/test/java/com/facebook/LinkBench/MySqlNodeStoreTest.java index e0bc97c0..8ecfaa9c 100644 --- a/src/test/java/com/facebook/LinkBench/MySqlNodeStoreTest.java +++ b/src/test/java/com/facebook/LinkBench/MySqlNodeStoreTest.java @@ -23,7 +23,7 @@ public class MySqlNodeStoreTest extends NodeStoreTestBase { Connection conn; Properties currProps; - + @Override protected Properties basicProps() { Properties props = super.basicProps(); diff --git a/src/test/java/com/facebook/LinkBench/MySqlTestConfig.java b/src/test/java/com/facebook/LinkBench/MySqlTestConfig.java index 51d2f21d..903bf66b 100644 --- a/src/test/java/com/facebook/LinkBench/MySqlTestConfig.java +++ b/src/test/java/com/facebook/LinkBench/MySqlTestConfig.java @@ -27,7 +27,7 @@ * @author tarmstrong */ public class MySqlTestConfig { - + // Hardcoded parameters for now static String host = "localhost"; static int port = 3306; @@ -36,7 +36,7 @@ public class MySqlTestConfig { static String linktable = "test_linktable"; static String counttable = "test_counttable"; static String nodetable = "test_nodetable"; - + public static void fillMySqlTestServerProps(Properties props) { props.setProperty(Config.LINKSTORE_CLASS, LinkStoreMysql.class.getName()); props.setProperty(Config.NODESTORE_CLASS, LinkStoreMysql.class.getName()); @@ -49,12 +49,12 @@ public static void fillMySqlTestServerProps(Properties props) { props.setProperty(Config.NODE_TABLE, nodetable); } - static Connection createConnection(String testDB) + static Connection createConnection(String testDB) throws InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException { Class.forName("com.mysql.jdbc.Driver").newInstance(); return DriverManager.getConnection( - "jdbc:mysql://"+ MySqlTestConfig.host + ":" + + "jdbc:mysql://"+ MySqlTestConfig.host + ":" + MySqlTestConfig.port + "/" + testDB + "?elideSetAutoCommits=true" + "&useLocalTransactionState=true" + @@ -62,13 +62,13 @@ static Connection createConnection(String testDB) "&useLocalSessionState=true", MySqlTestConfig.user, MySqlTestConfig.pass); } - - static void createTestTables(Connection conn, String testDB) + + static void createTestTables(Connection conn, String testDB) throws SQLException { Statement stmt = conn.createStatement(); stmt.executeUpdate(String.format( - "CREATE TABLE `%s`.`%s` (" + + "CREATE TABLE `%s`.`%s` (" + "`id1` bigint(20) unsigned NOT NULL DEFAULT '0'," + "`id2` bigint(20) unsigned NOT NULL DEFAULT '0'," + "`link_type` bigint(20) unsigned NOT NULL DEFAULT '0'," + @@ -78,7 +78,7 @@ static void createTestTables(Connection conn, String testDB) "`version` int(11) unsigned NOT NULL DEFAULT '0'," + "PRIMARY KEY (`id1`,`id2`,`link_type`)," + "KEY `id1_type` (`id1`,`link_type`,`visibility`,`time`,`version`,`data`)" + - ") ENGINE=InnoDB DEFAULT CHARSET=latin1;", + ") ENGINE=InnoDB DEFAULT CHARSET=latin1;", testDB, MySqlTestConfig.linktable)); stmt.executeUpdate(String.format("CREATE TABLE `%s`.`%s` (" + "`id` bigint(20) unsigned NOT NULL DEFAULT '0'," + @@ -90,18 +90,18 @@ static void createTestTables(Connection conn, String testDB) ") ENGINE=InnoDB DEFAULT CHARSET=latin1;", testDB, MySqlTestConfig.counttable)); stmt.executeUpdate(String.format( - "CREATE TABLE `%s`.`%s` (" + + "CREATE TABLE `%s`.`%s` (" + "`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT," + "`type` int(10) unsigned NOT NULL," + "`version` bigint(20) unsigned NOT NULL," + "`time` int(10) unsigned NOT NULL," + "`data` mediumtext NOT NULL," + "primary key(`id`)" + - ") ENGINE=InnoDB DEFAULT CHARSET=latin1;", + ") ENGINE=InnoDB DEFAULT CHARSET=latin1;", testDB, MySqlTestConfig.nodetable)); } - static void dropTestTables(Connection conn, String testDB) + static void dropTestTables(Connection conn, String testDB) throws SQLException { Statement stmt = conn.createStatement(); stmt.executeUpdate(String.format("DROP TABLE IF EXISTS `%s`.`%s`;", diff --git a/src/test/java/com/facebook/LinkBench/NodeStoreTestBase.java b/src/test/java/com/facebook/LinkBench/NodeStoreTestBase.java index 57cac39b..4e240813 100644 --- a/src/test/java/com/facebook/LinkBench/NodeStoreTestBase.java +++ b/src/test/java/com/facebook/LinkBench/NodeStoreTestBase.java @@ -24,48 +24,48 @@ import org.junit.Test; /** - * This test implements unit tests that *all* implementations of NodeStore + * This test implements unit tests that *all* implementations of NodeStore * should pass. - * + * * Different implementations of NodeStore will require different configuration - * and different setups for testing, so in order to test out a particular + * and different setups for testing, so in order to test out a particular * NodeStore implementation, you can subclass this test and implement the * required abstract methods so that the test store is initialized correctly * and all required configuration properties are filled in. - * + * * @author tarmstrong */ public abstract class NodeStoreTestBase extends TestCase { protected String testDB = "linkbench_unittestdb"; - + protected abstract void initNodeStore(Properties props) throws Exception, IOException; - - protected abstract NodeStore getNodeStoreHandle(boolean initialized) + + protected abstract NodeStore getNodeStoreHandle(boolean initialized) throws Exception, IOException; - + protected Properties basicProps() { Properties props = new Properties(); props.setProperty(Config.DBID, testDB); return props; } - + @Override public void setUp() throws Exception { super.setUp(); Properties props = basicProps(); - + // Set up db initNodeStore(props); getNodeStoreHandle(true).resetNodeStore(testDB, 0); } - + @Test public void testIDAlloc() throws IOException, Exception { int now = (int)(System.currentTimeMillis()/1000L); NodeStore store = getNodeStoreHandle(true); - + Node test = new Node(-1, 2048, 1, now, new byte[] {0xb, 0xe, 0xa, 0x5, 0x7}); store.resetNodeStore(testDB, 4); // We always start counting from 4 at Facebook long id = store.addNode(testDB, test); @@ -73,25 +73,25 @@ public void testIDAlloc() throws IOException, Exception { assertEquals(4, id); assertEquals(-1, test.id); // Check not modified test.id = id; - + // Allocate another id = store.addNode(testDB, test); test.id = id; assertEquals(5, id); - + // Check retrieval Node fetched = store.getNode(testDB, 2048, 5); assertTrue(fetched != test); // should not alias assertTrue(fetched + ".equals(" + test + ")", fetched.equals(test)); // but should have same data - + // Check deletion assertTrue(store.deleteNode(testDB, 2048, 5)); assertNull(store.getNode(testDB, 2048, 5)); // Delete non-existent data assertFalse(store.deleteNode(testDB, 2048, 8)); // bad id assertFalse(store.deleteNode(testDB, 2049, 4)); // bad type - + // Check reset works right store.resetNodeStore(testDB, 3); assertNull(store.getNode(testDB, 2048, 4)); @@ -100,47 +100,47 @@ public void testIDAlloc() throws IOException, Exception { assertNotNull(store.getNode(testDB, 2048, 3)); assertNotNull(store.getNode(testDB, 2048, 4)); } - + @Test public void testUpdate() throws IOException, Exception { NodeStore store = getNodeStoreHandle(true); store.resetNodeStore(testDB, 0); - + Node test = new Node(-1, 1234, 3, 3, "the quick brown fox".getBytes()); test.id = store.addNode(testDB, test); test.data = "jumped over the lazy dog".getBytes(); assertTrue(store.updateNode(testDB, test)); - + Node test2 = store.getNode(testDB, test.type, test.id); assertNotNull(test2); assertTrue(test.equals(test2)); } - + @Test public void testBinary() throws IOException, Exception { byte data[] = new byte[4096]; - + for (int i = 0; i < data.length; i++) { data[i] = (byte)(i % 256); } - + NodeStore store = getNodeStoreHandle(true); store.resetNodeStore(testDB, 0); - + Node test = new Node(-1, 1234, 3, 3, data); test.id = store.addNode(testDB, test); - + Node test2 = store.getNode(testDB, test.type, test.id); assertNotNull(test2); assertTrue(Arrays.equals(data, test2.data)); - + byte data2[] = new byte[data.length * 2]; for (int i = 0; i < data2.length; i++) { data2[i] = (byte)((i + 52) % 256); } - test.data = data2; + test.data = data2; assertTrue(store.updateNode(testDB, test)); - + Node test3 = store.getNode(testDB, test.type, test.id); assertNotNull(test3); assertTrue(Arrays.equals(data2, test3.data)); diff --git a/src/test/java/com/facebook/LinkBench/PiecewiseDistTest.java b/src/test/java/com/facebook/LinkBench/PiecewiseDistTest.java index 97ff42ae..bad42466 100644 --- a/src/test/java/com/facebook/LinkBench/PiecewiseDistTest.java +++ b/src/test/java/com/facebook/LinkBench/PiecewiseDistTest.java @@ -46,7 +46,7 @@ public void setUp() throws Exception { testDistribution.add(new Point(110, 0.92)); testDistribution.add(new Point(120, 1.0)); } - + @Override protected int cdfChecks() { return 50; @@ -62,25 +62,25 @@ public void init(long min, long max, Properties props, String keyPrefix) { } }; } - + @Override public void testCDFSanity() { System.err.println("CDF not implemented"); } - + @Override public void testCDFChooseConsistency() { System.err.println("CDF not implemented"); } - + @Override public void testCDFPDFConsistency() { System.err.println("CDF not implemented"); } - + @Override public void testQuantileSanity() { System.err.println("Quantile not implemented"); } - + } diff --git a/src/test/java/com/facebook/LinkBench/TestAccessDistribution.java b/src/test/java/com/facebook/LinkBench/TestAccessDistribution.java index 46f20e65..1bee2591 100644 --- a/src/test/java/com/facebook/LinkBench/TestAccessDistribution.java +++ b/src/test/java/com/facebook/LinkBench/TestAccessDistribution.java @@ -58,7 +58,7 @@ public void testReciprocal() { public void testRoundRobin() { testSanityBuiltinDist(AccessDistMode.ROUND_ROBIN, 0); } - + @Test public void testUniform() { UniformDistribution u = new UniformDistribution(); @@ -67,8 +67,8 @@ public void testUniform() { u.init(min, max, props, ""); ProbAccessDistribution unshuffled = new ProbAccessDistribution(u, null); testSanityAccessDist(unshuffled, min, max); - - ProbAccessDistribution shuffled = new ProbAccessDistribution(u, + + ProbAccessDistribution shuffled = new ProbAccessDistribution(u, new InvertibleShuffler(13, 25, max - min)); testSanityAccessDist(shuffled, min, max); } @@ -82,30 +82,30 @@ public void testZipf() { z.init(min, max, props, ""); ProbAccessDistribution unshuffled = new ProbAccessDistribution(z, null); testSanityAccessDist(unshuffled, min, max); - - ProbAccessDistribution shuffled = new ProbAccessDistribution(z, + + ProbAccessDistribution shuffled = new ProbAccessDistribution(z, new InvertibleShuffler(13, 25, max - min)); testSanityAccessDist(shuffled, min, max); } - - + + @Test public void testReal() { RealDistribution r = new RealDistribution(); Properties props = new Properties(); - props.setProperty(Config.DISTRIBUTION_DATA_FILE, + props.setProperty(Config.DISTRIBUTION_DATA_FILE, new File("config/Distribution.dat").getAbsolutePath()); int min = 100, max = 200; r.init(props, min, max, DistributionType.LINK_READS); ProbAccessDistribution unshuffled = new ProbAccessDistribution(r, null); testSanityAccessDist(unshuffled, min, max); - - ProbAccessDistribution shuffled = new ProbAccessDistribution(r, + + ProbAccessDistribution shuffled = new ProbAccessDistribution(r, new InvertibleShuffler(13, 25, max - min)); testSanityAccessDist(shuffled, min, max); } - - + + public static void testSanityBuiltinDist(AccessDistMode mode, long config) { long minid = 123; @@ -132,7 +132,7 @@ public static void testSanityAccessDist(AccessDistribution dist, long minid, assertTrue(id >= minid); assertTrue(id < maxid); } catch (AssertionFailedError e) { - System.err.println("Error: on trial " + i + " id returned: " + System.err.println("Error: on trial " + i + " id returned: " + id + " not in range [" + minid + "," + maxid + ")"); throw e; } diff --git a/src/test/java/com/facebook/LinkBench/TestDataGen.java b/src/test/java/com/facebook/LinkBench/TestDataGen.java index 226046c2..59e433b5 100644 --- a/src/test/java/com/facebook/LinkBench/TestDataGen.java +++ b/src/test/java/com/facebook/LinkBench/TestDataGen.java @@ -30,7 +30,7 @@ import com.facebook.LinkBench.generators.UniformDataGenerator; public class TestDataGen extends TestCase { - + public static void printByteGrid(byte[] data) { for (int i = 0; i < data.length; i += 32) { for (int j = i; j < Math.min(i + 32, data.length); j++) { @@ -39,7 +39,7 @@ public static void printByteGrid(byte[] data) { System.err.println(); } } - + /** * Test how quickly uniform data generator can generate patterns */ @@ -57,10 +57,10 @@ public DataGenerator make(double param) { testTiming(fact, 128); testTiming(fact, 1024); } - + /** * Test how quickly motif data generator can generate patterns to make - * sure its not + * sure its not */ @Test public void testTimingMotif() { @@ -76,11 +76,11 @@ public DataGenerator make(double param) { testTiming(fact, 128); testTiming(fact, 1024); } - + private void testTiming(DataGenFactory fact, int bufSize) { byte buf[] = new byte[bufSize]; Random rng = new Random(); - + int trials = 200000; double params[] = new double[] {0.0, 0.25, 0.5, 0.75, 1.0}; long times_ns[] = new long[params.length]; @@ -97,23 +97,23 @@ private void testTiming(DataGenFactory fact, int bufSize) { long timeTaken = doTest(fact, buf, rng, trials, param); times_ns[i] = timeTaken; } - + for (int i = 0; i < params.length; i++) { double trialTime = times_ns[i] / (double) trials; double byteTime = trialTime / buf.length; System.err.format("uniqueness = %.3f, time for %d byte buffer = %.1f ns, time per byte = %.1fns\n", - params[i], buf.length, trialTime, byteTime); - + params[i], buf.length, trialTime, byteTime); + } } private static interface DataGenFactory { public abstract DataGenerator make(double param); } - + private long doTest(DataGenFactory fact, byte[] buf, Random rng, int trials, double param) { DataGenerator gen = fact.make(param); - + long start = System.nanoTime(); for (int j = 0; j < trials; j++) { gen.fill(rng, buf); @@ -122,41 +122,41 @@ private long doTest(DataGenFactory fact, byte[] buf, Random rng, int trials, dou long timeTaken = end - start; return timeTaken; } - + /** * Exercise the motif data generator and print the output. - * + * * Currently difficult to automatically verify output. */ @Test public void testMotif() { MotifDataGenerator gen = new MotifDataGenerator(); - + System.err.println("uniqueness 0.25"); gen.init(0, 8, 0.25); byte data[] = gen.fill(new Random(), new byte[64]); printByteGrid(data); - + System.err.println("uniqueness 0.0"); gen.init(0, 8, 0.0); data = gen.fill(new Random(), new byte[64]); printByteGrid(data); - + System.err.println("uniqueness 0.05"); gen.init(0, 8, 0.05); data = gen.fill(new Random(), new byte[64]); printByteGrid(data); - + System.err.println("uniqueness 1.0"); gen.init(0, 8, 1.0); data = gen.fill(new Random(), new byte[64]); printByteGrid(data); } - + /** - * Estimate the compressibility of randomly generated data by + * Estimate the compressibility of randomly generated data by * compressing a long stream of the data - * @throws IOException + * @throws IOException */ @Test public void testCompressibility() throws IOException { @@ -165,28 +165,28 @@ public void testCompressibility() throws IOException { System.err.println("\nUniqueness=0.5 Range=255\n==============="); testCompressibility(gen, 1024, 10000); testCompressibility(gen, 64, 1); - + gen.init(0, 127, 0.5); System.err.println("\nUniqueness=0.5 Range=127\n==============="); testCompressibility(gen, 1024, 10000); testCompressibility(gen, 64, 1); - + gen.init(0, 255, 0.0); System.err.println("\nUniqueness=0.0 Range=255\n==============="); testCompressibility(gen, 1024, 10000); testCompressibility(gen, 64, 1); - + gen.init(0, 255, 1.0); System.err.println("\nUniqueness=1.0 Range=255\n==============="); testCompressibility(gen, 1024, 10000); testCompressibility(gen, 64, 1); gen.init(0, 255, 1.0); - + gen.init(0, 127, 1.0); System.err.println("\nUniqueness=1.0 Range=127\n==============="); testCompressibility(gen, 1024, 10000); testCompressibility(gen, 64, 1); - + gen.init(0, 1, 1.0); System.err.println("\nUniqueness=1.0 Range=1\n==============="); testCompressibility(gen, 1024, 10000); @@ -201,14 +201,14 @@ private void testCompressibility(MotifDataGenerator gen, int blockSize, int bloc ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); Deflater def = new Deflater(Deflater.BEST_COMPRESSION); DeflaterOutputStream gzipOut = new DeflaterOutputStream(byteOut, def); - + byte block[] = new byte[blockSize]; for (int i = 0; i < blocks; i++) { gen.fill(rng, block); gzipOut.write(block); } gzipOut.close(); - + byte compressed[] = byteOut.toByteArray(); int origLen = blockSize * blocks; System.err.format("%dx%d blocks. Compressed %d bytes to %d: %.2f. Bound: %.2f\n", diff --git a/src/test/java/com/facebook/LinkBench/TestRealDistribution.java b/src/test/java/com/facebook/LinkBench/TestRealDistribution.java index ce0c985f..aee433ec 100644 --- a/src/test/java/com/facebook/LinkBench/TestRealDistribution.java +++ b/src/test/java/com/facebook/LinkBench/TestRealDistribution.java @@ -32,13 +32,13 @@ import com.facebook.LinkBench.distributions.LinkDistributions.ProbLinkDistribution; /* - * This class measures how similar data generated by class RealDistribution + * This class measures how similar data generated by class RealDistribution * and the data file is. */ public class TestRealDistribution extends TestCase { Properties props; - + @Override public void setUp() throws Exception { props = new Properties(); @@ -46,10 +46,10 @@ public void setUp() throws Exception { props.setProperty(Config.DISTRIBUTION_DATA_FILE, distFile); RealDistribution.loadOneShot(props); } - + /** regression test for binary search bug */ public void testBinSearchRegression1() { - ArrayList l = + ArrayList l = new ArrayList(); l.add(new RealDistribution.Point(0, 0.0)); l.add(new RealDistribution.Point(1, 0.5)); @@ -83,32 +83,32 @@ public void testGetNLinks() throws Exception { //TODO: would be good to have some real measure for whether these // error values are in the "proper" range System.out.println("testGetNLinks\n==========="); - + double err; err = testGetNlinks(props, 1000000, 2000001); System.out.println("testGetNlinks(1000000, 2000001) err=" + err); assertTrue(err < 0.0001); - + err = testGetNlinks(props, 1234567, 7654321); System.out.println("testGetNlinks(1234567, 7654321) err=" + err); assertTrue(err < 0.0001); - + err = testGetNlinks(props, 97, 10000097); System.out.println("testGetNlinks(97, 10000097) err=" + err); assertTrue(err < 0.0001); System.out.println(); } - + @Test public void testGetNextId1() throws Exception { //TODO: would be good to have some real measure for whether these // error values are in the "proper" range System.out.println("testGetNextId1\n==========="); - + long randSeed = System.currentTimeMillis(); System.out.println("random seed: " + randSeed); Random rng = new Random(randSeed); - + double err; err = testGetNextId1(props, rng, 1000000, 2000001, DistributionType.LINK_READS); System.out.println("testGetNextId1(1000000, 2000001, nreads) err=" + err); @@ -123,15 +123,15 @@ public void testGetNextId1() throws Exception { System.out.println("testGetNextId1(1234567, 7654321, nreads) err=" + err); err = testGetNextId1(props, rng, 1234567, 7654321, DistributionType.LINK_WRITES); System.out.println("testGetNextId1(1234567, 7654321, nwrites) err=" + err); - + err = testGetNextId1(props, rng, 97, 10000097, DistributionType.LINK_READS); - + System.out.println("testGetNextId1(97, 10000097, nreads) err=" + err); err = testGetNextId1(props, rng, 97, 10000097, DistributionType.LINK_WRITES); System.out.println("testGetNextId1(97, 10000097, nwrites) err=" + err); System.out.println(); } - + //return the distribution for a sequence of numbers private static NavigableMap getDistribution(int[] seq, int start, int end) { @@ -160,8 +160,8 @@ private static double getComparisonError(NavigableMap act, int max = Math.min(act.lastKey(), exp.lastKey()); int samplePoints = Math.min(10000, max - min + 1); double cumulative = 0; - - // Sample linearly interpolated distribution at many points + + // Sample linearly interpolated distribution at many points for (int i = 0; i < samplePoints; i++) { int k = min + (int)Math.round(((max - min) * ((double)i)/((double)samplePoints))); @@ -172,7 +172,7 @@ private static double getComparisonError(NavigableMap act, // Print errors > 0.1% if (printBigErrors && Math.abs(pa - pe) > 0.001) { System.err.println(String.format("Large divergence %f " - + "cdf_exp(%d) = %f, cdf_act(%d) = %f", + + "cdf_exp(%d) = %f, cdf_act(%d) = %f", Math.abs(pa - pe), k, pe, k, pa)); } } @@ -185,7 +185,7 @@ private static double interpolatedValue(NavigableMap a, int k) if (ceil.getKey() == floor.getKey()) { return ceil.getValue(); } - double mix = (k - floor.getKey()) / + double mix = (k - floor.getKey()) / (double)(ceil.getKey() - floor.getKey()); return mix * ceil.getValue() + (1 - mix) * floor.getValue(); } @@ -197,10 +197,10 @@ private static double testGetNextId1(Properties props, Random rng, int startid1, int maxid1, DistributionType type) throws Exception { int[] cnt = new int[maxid1]; - + double nqueries = (maxid1 - startid1)*RealDistribution.getArea(type) /100.0; - + RealDistribution dist = new RealDistribution(); dist.init(props, startid1, maxid1, type); for (int i = 0; i < nqueries; ++i) { @@ -210,7 +210,7 @@ private static double testGetNextId1(Properties props, Random rng, } cnt[(int)x]++; } - + NavigableMap generated_data = getDistribution(cnt, startid1, maxid1); NavigableMap real_data = RealDistribution.getCDF(type); @@ -219,7 +219,7 @@ private static double testGetNextId1(Properties props, Random rng, //test getNLinks private static double testGetNlinks(Properties props, - int startid1, int maxid1) + int startid1, int maxid1) throws Exception { RealDistribution rDist = new RealDistribution(); rDist.init(props, startid1, maxid1, DistributionType.LINKS); diff --git a/src/test/java/com/facebook/LinkBench/TestStats.java b/src/test/java/com/facebook/LinkBench/TestStats.java index 2f4f6f90..2232855c 100644 --- a/src/test/java/com/facebook/LinkBench/TestStats.java +++ b/src/test/java/com/facebook/LinkBench/TestStats.java @@ -23,7 +23,7 @@ import com.facebook.LinkBench.stats.LatencyStats; public class TestStats extends TestCase { - + @Test public void testBucketing() { // 0 microseconds until 100 seconds diff --git a/src/test/java/com/facebook/LinkBench/TimerTest.java b/src/test/java/com/facebook/LinkBench/TimerTest.java index 0a3ce3a4..72a693e4 100644 --- a/src/test/java/com/facebook/LinkBench/TimerTest.java +++ b/src/test/java/com/facebook/LinkBench/TimerTest.java @@ -22,7 +22,7 @@ import junit.framework.TestCase; public class TimerTest extends TestCase { - + @Test public void testTimer1() { for (int i = 0; i < 100; i++) { @@ -30,13 +30,13 @@ public void testTimer1() { Timer.waitUntil(wakeTime); long now = System.nanoTime(); assertTrue(now >= wakeTime); - + // Check that the precision isn't so awful that it would // indicate a definite bug (e.g. 100ms) assertTrue(now <= wakeTime + 1e7); } } - + /** * Test that we can use the timer to wait for short intervals * without getting far behind @@ -44,7 +44,7 @@ public void testTimer1() { @Test public void testTimer2() { // Repeatedly wait for 10us - + long waits = 100 * 100; // 100ms total long time = System.nanoTime(); long startTime = time; @@ -57,17 +57,17 @@ public void testTimer2() { assertTrue(endTime - startTime >= 1e8); assertTrue(endTime - startTime < 1.02e8); // no longer than 102ms } - + @Test public void testExponentialArrivals() { long randSeed = System.currentTimeMillis(); System.err.println("Random seed: " + randSeed); Random rng = new Random(randSeed); - + int trials = 40000; int arrivalRate_s = 200000; double arrivalRate_ns = arrivalRate_s / (double)1e9; - + // Check that the exponential distribution is creating correct arrival rate. long startTime = System.nanoTime(); long time = startTime; @@ -75,12 +75,12 @@ public void testExponentialArrivals() { time = Timer.waitExpInterval(rng, time, arrivalRate_ns); } long endTime = System.nanoTime(); - + double actualArrivalRate_ns = trials / (double) (endTime - startTime); - System.err.println("actual arrival rate: " + System.err.println("actual arrival rate: " + actualArrivalRate_ns * 1e9 + " /s " + - " expected " + arrivalRate_s + "/s"); - + " expected " + arrivalRate_s + "/s"); + assertTrue(actualArrivalRate_ns >= arrivalRate_ns * 0.95); assertTrue(actualArrivalRate_ns <= arrivalRate_ns * 1.05); } diff --git a/src/test/java/com/facebook/LinkBench/UniformDistTest.java b/src/test/java/com/facebook/LinkBench/UniformDistTest.java index 297e94e7..23f53ce1 100644 --- a/src/test/java/com/facebook/LinkBench/UniformDistTest.java +++ b/src/test/java/com/facebook/LinkBench/UniformDistTest.java @@ -28,14 +28,14 @@ public class UniformDistTest extends DistributionTestBase { public ProbabilityDistribution getDist() { return new UniformDistribution(); } - + @Test public void testInRange() { // Check 2^31 < n < 2^32 and n > 2^32 long maxes[] = {(long)Math.pow(2, 31.5), (long)Math.pow(2, 34.23)}; int trials = 10000; Random rng = new Random(); - + for (long max: maxes) { UniformDistribution dist = new UniformDistribution(); dist.init(0, max, 1); diff --git a/src/test/java/com/facebook/LinkBench/ZipfDistTest.java b/src/test/java/com/facebook/LinkBench/ZipfDistTest.java index c4a069e3..a7492cda 100644 --- a/src/test/java/com/facebook/LinkBench/ZipfDistTest.java +++ b/src/test/java/com/facebook/LinkBench/ZipfDistTest.java @@ -27,12 +27,12 @@ protected int cdfChecks() { // Don't do many checks, since its expensive return 5; } - + @Override public ProbabilityDistribution getDist() { return new ZipfDistribution(); } - + @Override public Properties getDistParams() { Properties props = new Properties(); @@ -40,7 +40,7 @@ public Properties getDistParams() { return props; } - + @Override protected Bucketer getBucketer() { return new ZipfBucketer(); @@ -48,7 +48,7 @@ protected Bucketer getBucketer() { @Override protected double tolerance() { - /* + /* * Method for choosing IDs isn't 100% precise * but something is more seriously wrong if it is more than 1% off */ @@ -59,14 +59,14 @@ protected double tolerance() { * Check distribution more closely at low values */ static class ZipfBucketer implements Bucketer { - private static final long bucketBounds[] = + private static final long bucketBounds[] = {0, 1, 2, 3, 4, 10, 20, 30, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000}; public int getBucketCount() { return bucketBounds.length + 1; } - + public int chooseBucket(long i, long n) { for (int j = 0; j < bucketBounds.length; j++) { if (i <= bucketBounds[j]) { @@ -75,7 +75,7 @@ public int chooseBucket(long i, long n) { } return bucketBounds.length; } - + public long bucketMax(int bucket, long n) { if (bucket >= bucketBounds.length) return n; else return bucketBounds[bucket];