Permalink
Browse files

Adding assoc schema APIs for rocksdb so that linkbench can run rocksdb.

Summary:
LinkStoreRocksDb.java contains the assoc-schema APIs that enable
linkbench to be run over rocksdb. swift has been incorporated to make the thrift
calls using a java client. It uses netty and is faster. Am still working on it. Checking it in so that it can be tested by others

Test Plan: ~/linkbench/bin/linkbench -c config/LinkConfigRocksDb.properties –l; <Run the rocks server in fbcode simultaneously>

Reviewers: dhruba, tarmstrong

Reviewed By: tarmstrong

CC: heyongqiang, sheki, zshao, kosievdmerwe, vamsi, andrewcox

Differential Revision: https://reviews.facebook.net/D8115
  • Loading branch information...
1 parent 6438dae commit 7481a67cfad296eed94bd93ac8d3511c566c536b @emayanke emayanke committed Jan 3, 2013
View
@@ -84,7 +84,7 @@ Prerequisites:
These instructions assume you are using a UNIX-like system such as a Linux distribution
or Mac OS X.
-**Java**: You will need a Java 6+ runtime environment. LinkBench by default
+**Java**: You will need a Java 7+ runtime environment. LinkBench by default
uses the version of Java on your path. You can override this by setting the
JAVA\_HOME environment variable to the directory of the desired
Java runtime version. You will also need a Java JDK to compile from source.
View
@@ -71,6 +71,11 @@ for f in $LINKBENCH_HOME/lib/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
+# add libs to CLASSPATH
+for f in $LINKBENCH_HOME/lib/build/lib/*.jar; do
+ CLASSPATH=${CLASSPATH}:$f;
+done
+
# add latest jar to CLASSPATH
CLASSPATH=${CLASSPATH}:dist/FacebookLinkBench.jar;
View
@@ -0,0 +1,12 @@
+version=0.1-SNAPSHOT
+
+linkbench.groupId=com.facebook.linkbench
+linkbench.artifactId=linkbench
+linkbench.url=https://github.com/facebook/linkbench
+
+mvn.fb.repo=http://repo1.maven.org/maven2
+mvn.ant.task.url=${mvn.fb.repo}/org/apache/maven/maven-ant-tasks/${mvn.ant.task.version}
+mvn.ant.task.jar=maven-ant-tasks-${mvn.ant.task.version}.jar
+mvn.ant.task.version=2.1.3
+
+swift.version=0.2.0
View
@@ -1,19 +1,24 @@
-<project name="FacebookLinkBench" default="dist" basedir=".">
+<project name="FacebookLinkBench" default="dist" basedir="."
+ xmlns:artifact="antlib:org.apache.maven.artifact.ant">
<property name="linkbench.version" value="0.1"/>
<description>
simple example build file
</description>
+
<!-- set global properties for this build -->
+ <property file="${basedir}/build.properties" />
<property name="src" location="src"/>
<property name="build" location="build"/>
+ <property name="build.lib" location="build/lib"/>
+ <property name="build.tools" location="build/tools"/>
<property name="dist" location="dist"/>
<property name="lib" location="lib"/>
<property name="test.output" location="test-output"/>
<property name="test.rawoutput" location="${test.output}/raw"/>
<property name="test.tmp" location="${test.output}/tmp"/>
<property name="test.report" location="${test.output}/report"/>
- <property name="linkbench.jar"
+ <property name="linkbench.jar"
value="${dist}/FacebookLinkBench-${linkbench.version}.jar"/>
<property name="linkbench.latest.jar" value="${dist}/FacebookLinkBench.jar"/>
@@ -26,16 +31,22 @@
<tstamp/>
<!-- Create the build directory structure used by compile -->
<mkdir dir="${build}"/>
+ <mkdir dir="${build.lib}"/>
+ <mkdir dir="${build.tools}"/>
</target>
- <target name="compile" depends="init"
+ <target name="compile" depends="init,mvn.init"
description="compile the source " >
<!-- Compile the java code from ${src} into ${build} -->
<javac includeantruntime="false" srcdir="${src}" destdir="${build}"
+ source="1.7" target="1.7"
debug="${debug}">
<classpath>
<pathelement path="${classpath}"/>
- <fileset dir="lib">
+ <fileset dir="${lib}">
+ <include name="**/*.jar"/>
+ </fileset>
+ <fileset dir="${build.lib}">
<include name="**/*.jar"/>
</fileset>
<pathelement location="classes"/>
@@ -73,7 +84,7 @@
<assertions> <enable /> </assertions>
<batchtest todir="${test.rawoutput}">
- <fileset dir="${build}">
+ <fileset dir="${build}">
<include name="**/*Test*.class"/>
<exclude name="**/LinkStoreHBaseGeneralAtomicityTesting.class"/>
<exclude name="**/*TestBase.class"/>
@@ -100,4 +111,41 @@
<delete dir="${dist}"/>
<delete dir="${test.output}"/>
</target>
+
+ <target name="proxy" if="proxy.enabled">
+ <setproxy proxyhost="${proxy.host}" proxyport="${proxy.port}"
+ proxyuser="${proxy.user}" proxypassword="${proxy.pass}"/>
+ </target>
+
+ <target name="mvn.ant.tasks.download" depends="init,mvn.ant.tasks.check,proxy" unless="mvn.ant.tasks.found">
+ <get src="${mvn.ant.task.url}/${mvn.ant.task.jar}" dest="${build.tools}/${mvn.ant.task.jar}" usetimestamp="true"/>
+ </target>
+
+ <target name="mvn.ant.tasks.check">
+ <condition property="mvn.ant.tasks.found">
+ <typefound uri="antlib:org.apache.maven.artifact.ant" name="artifact"/>
+ </condition>
+ </target>
+
+ <target name="mvn.init" depends="mvn.ant.tasks.download" unless="mvn.finished">
+ <!-- Download mvn ant tasks, download dependencies, and setup pom file -->
+ <typedef uri="antlib:org.apache.maven.artifact.ant" classpath="${build.tools}/${mvn.ant.task.jar}"/>
+
+ <!-- remote repositories used to download dependencies from -->
+ <artifact:remoteRepository id="fbnexus" url="${mvn.fb.repo}"/>
+
+ <!-- Download the dependencies -->
+ <artifact:dependencies filesetId="build-dependency-jars" settingsFile="${basedir}/settings.xml">
+ <pom file="pom.xml"/>
+ </artifact:dependencies>
+
+ <!-- Copy the dependencies to the build/lib dir -->
+ <copy todir="${build}/lib">
+ <fileset refid="build-dependency-jars"/>
+ <mapper type="flatten"/>
+ </copy>
+
+ <property name="mvn.finished" value="true"/>
+ </target>
+
</project>
@@ -0,0 +1,111 @@
+# Sample RocksDb LinkBench configuration file.
+#
+# This file contains settings for the data store, as well as controlling
+# benchmark output and behavior. The workload is defined in a separate
+# file.
+#
+
+##########################
+# Workload Configuration #
+##########################
+
+# Path for workload properties file. Properties in this file will override
+# those in workload properties file.
+# Can be absolute path, or relative path from LinkBench home directory
+workload_file = config/FBWorkload.properties
+
+#################################
+# #
+# Data Source Configuration #
+# #
+#################################
+
+# Implementation of LinkStore and NodeStore to use
+linkstore = com.facebook.LinkBench.LinkStoreRocksDb
+nodestore = com.facebook.LinkBench.LinkStoreRocksDb
+
+# RocksDb connection information
+host = yourhostname.here
+port = 9090
+# dbid: the database name to use
+dbid = linkdb
+
+###############################
+# #
+# Logging and Stats Setup #
+# #
+###############################
+
+# This controls logging output. Settings are, in order of increasing
+# verbosity:
+# ERROR: only output serious errors
+# WARN: output warnings
+# INFO: output additional information such as progress
+# DEBUG: output high-level debugging information
+# TRACE: output more detailed lower-level debugging information
+debuglevel = INFO
+
+# display frequency of per-thread progress in seconds
+progressfreq = 300
+
+# display frequency of per-thread stats (latency, etc) in seconds
+displayfreq = 1800
+
+# display global load update (% complete, etc) after this many links loaded
+load_progress_interval = 50000
+
+# display global update on request phase (% complete, etc) after this many ops
+req_progress_interval = 10000
+
+# max number of samples to store for each per-thread statistic
+maxsamples = 10000
+
+###############################
+# #
+# Load Phase Configuration #
+# #
+###############################
+
+# number of threads to run during load phase
+loaders = 10
+
+# whether to generate graph nodes during load process
+generate_nodes = true
+
+# partition loading work into chunks of id1s of this size
+loader_chunk_size = 2048
+
+# seed for initial data load random number generation (optional)
+# load_random_seed = 12345
+
+##################################
+# #
+# Request Phase Configuration #
+# #
+##################################
+
+# number of threads to run during request phase
+requesters = 100
+
+# read + write requests per thread
+requests = 500000
+
+# request rate per thread. <= 0 means unthrottled requests, > 0 limits
+# the average request rate to that number of requests per second per thread,
+# with the inter-request intervals governed by an exponential distribution
+requestrate = 0
+
+# max duration in seconds for request phase of benchmark
+maxtime = 100000
+
+# warmup time in seconds. The benchmark is run for a warmup period
+# during which no statistics are recorded. This allows database caches,
+# etc to warm up.
+warmup_time = 0
+
+# seed for request random number generation (optional)
+# request_random_seed = 12345
+
+# maximum number of failures per requester to tolerate before aborting
+# negative number means never abort
+max_failed_requests = 100
View
Binary file not shown.
View
90 pom.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>com.facebook.linkbench</groupId>
+ <artifactId>linkbench</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ <packaging>pom</packaging>
+
+ <url>https://github.com/facebook/linkbench</url>
+ <inceptionYear>2012</inceptionYear>
+
+ <developers>
+ <developer>
+ <id>tarmstrong</id>
+ <name>Tim Armstrong</name>
+ <email>tarmstrong@fb.com</email>
+ </developer>
+ <developer>
+ <id>dhruba</id>
+ <name>Dhruba Borthakur</name>
+ <email>dhruba@fb.com</email>
+ </developer>
+ <developer>
+ <id>amayank</id>
+ <name>Mayank Agarwal</name>
+ <email>amayank@fb.com</email>
+ </developer>
+ <developer>
+ <id>andrewcox</id>
+ <name>Andrew Cox</name>
+ <email>andrewcox@fb.com</email>
+ </developer>
+ </developers>
+
+ <scm>
+ <connection>scm:git:https://github.com/facebook/linkbench.git</connection>
+ <developerConnection>scm:git@github.com:facebook/linkbench.git</developerConnection>
+ <url>https://github.com/facebook/linkbench</url>
+ <tag>HEAD</tag>
+ </scm>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.facebook.swift</groupId>
+ <artifactId>swift-codec</artifactId>
+ <version>0.2.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.facebook.swift</groupId>
+ <artifactId>swift-service</artifactId>
+ <version>0.2.0</version>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+ <profile>
+ <id>nexus</id>
+ <!--Enable snapshots for the built in central repo to direct -->
+ <!--all requests to nexus via the mirror -->
+ <repositories>
+ <repository>
+ <id>central</id>
+ <url>http://repo1.maven.org/maven2</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+ <pluginRepositories>
+ <pluginRepository>
+ <id>central</id>
+ <url>http://repo1.maven.org/maven2</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </pluginRepository>
+ </pluginRepositories>
+ </profile>
+ </profiles>
+
+</project>
@@ -215,7 +215,6 @@ void load() throws IOException, InterruptedException, Throwable {
logger.info("Skipping load data per the cmdline arg");
return;
}
-
// load data
int nLinkLoaders = ConfigUtil.getInt(props, Config.NUM_LOADERS);
@@ -259,10 +258,8 @@ void load() throws IOException, InterruptedException, Throwable {
loaders.add(new NodeLoader(props, logger, nodeStore, rng,
latencyStats, csvStreamFile, loaderId));
}
-
enqueueLoadWork(chunk_q, startid1, maxid1, nLinkLoaders,
new Random(masterRandom.nextLong()));
-
// run loaders
loadTracker.startTimer();
long loadTime = concurrentExec(loaders);
@@ -289,7 +286,7 @@ void load() throws IOException, InterruptedException, Throwable {
logger.info(String.format("LOAD PHASE COMPLETED. " +
" Loaded %d nodes (Expected %d)." +
" Loaded %d links (%.2f links per node). " +
- " Took %.1f seconds. Links/second = %d",
+ " Took %.1f seconds. Links/second = %d",
actualNodes, expectedNodes, actualLinks,
actualLinks / (double) actualNodes, loadTime_s,
(long) Math.round(actualLinks / loadTime_s)));
@@ -382,7 +379,6 @@ void sendrequests() throws IOException, InterruptedException, Throwable {
progress, new Random(masterRandom.nextLong()), i, nrequesters);
requesters.add(l);
}
-
progress.startTimer();
// run requesters
concurrentExec(requesters);
@@ -464,7 +460,6 @@ void drive() throws IOException, InterruptedException, Throwable {
public static void main(String[] args)
throws IOException, InterruptedException, Throwable {
processArgs(args);
-
LinkBenchDriver d = new LinkBenchDriver(configFile,
cmdLineProps, logFile);
try {
@@ -205,6 +205,7 @@ public void run() {
LoadChunk chunk;
try {
chunk = chunk_q.take();
+ //logger.info("chunk end="+chunk.end);
} catch (InterruptedException ie) {
logger.warn("InterruptedException not expected, try again", ie);
continue;
@@ -468,7 +469,6 @@ private void loadLink(Link link, long outlink_ix, long nlinks,
private void loadLinks(ArrayList<Link> loadBuffer) {
long timestart = System.nanoTime();
-
try {
// no inverses for now
int nlinks = loadBuffer.size();
Oops, something went wrong.

0 comments on commit 7481a67

Please sign in to comment.