Permalink
Browse files

Merge branch 'master' of http://github.com/brianfrankcooper/YCSB

  • Loading branch information...
2 parents f56ec1a + a7bbbc9 commit ee88bbf7e80dc1575d8041889e87264240ddc2ff @johanoskarsson johanoskarsson committed May 22, 2010
Showing with 1,778 additions and 477 deletions.
  1. +2 −1 .gitignore
  2. +8 −0 README
  3. +11 −1 build.xml
  4. +1 −0 db/cassandra-0.5/lib/README
  5. +3 −1 db/cassandra-0.5/src/com/yahoo/ycsb/db/CassandraClient5.java
  6. +1 −0 db/cassandra-0.6/lib/README
  7. +3 −1 db/cassandra-0.6/src/com/yahoo/ycsb/db/CassandraClient6.java
  8. +1 −0 db/cassandra-0.7/lib/README
  9. +3 −1 db/cassandra-0.7/src/com/yahoo/ycsb/db/CassandraClient7.java
  10. +3 −0 db/hbase/src/com/yahoo/ycsb/db/HBaseClient.java
  11. +1 −0 db/mongodb/lib/README
  12. +285 −0 db/mongodb/src/com/yahoo/ycsb/db/MongoDbClient.java
  13. +2 −2 doc/coreproperties.html
  14. +2 −2 doc/coreworkloads.html
  15. +3 −3 doc/dblayer.html
  16. BIN doc/images/ycsb.jpg
  17. +2 −2 doc/index.html
  18. +27 −14 doc/javadoc/allclasses-frame.html
  19. +27 −14 doc/javadoc/allclasses-noframe.html
  20. +6 −3 doc/javadoc/com/yahoo/ycsb/BasicDB.html
  21. +8 −5 doc/javadoc/com/yahoo/ycsb/Client.html
  22. +26 −11 doc/javadoc/com/yahoo/ycsb/DB.html
  23. +6 −3 doc/javadoc/com/yahoo/ycsb/DBException.html
  24. +6 −3 doc/javadoc/com/yahoo/ycsb/DBFactory.html
  25. +8 −5 doc/javadoc/com/yahoo/ycsb/DBWrapper.html
  26. +8 −5 doc/javadoc/com/yahoo/ycsb/UnknownDBException.html
  27. +6 −3 doc/javadoc/com/yahoo/ycsb/Utils.html
  28. +6 −3 doc/javadoc/com/yahoo/ycsb/Workload.html
  29. +8 −5 doc/javadoc/com/yahoo/ycsb/WorkloadException.html
  30. +32 −7 doc/javadoc/com/yahoo/ycsb/db/HBaseClient.html
  31. +10 −4 doc/javadoc/com/yahoo/ycsb/db/package-frame.html
  32. +23 −8 doc/javadoc/com/yahoo/ycsb/db/package-summary.html
  33. +9 −5 doc/javadoc/com/yahoo/ycsb/db/package-tree.html
  34. +4 −28 doc/javadoc/com/yahoo/ycsb/package-frame.html
  35. +10 −55 doc/javadoc/com/yahoo/ycsb/package-summary.html
  36. +10 −12 doc/javadoc/com/yahoo/ycsb/package-tree.html
  37. +6 −3 doc/javadoc/com/yahoo/ycsb/workloads/CoreWorkload.html
  38. +2 −2 doc/javadoc/com/yahoo/ycsb/workloads/package-frame.html
  39. +8 −5 doc/javadoc/com/yahoo/ycsb/workloads/package-summary.html
  40. +8 −4 doc/javadoc/com/yahoo/ycsb/workloads/package-tree.html
  41. +253 −51 doc/javadoc/constant-values.html
  42. +6 −2 doc/javadoc/deprecated-list.html
  43. +6 −2 doc/javadoc/help-doc.html
  44. +410 −147 doc/javadoc/index-all.html
  45. +1 −1 doc/javadoc/index.html
  46. +9 −3 doc/javadoc/overview-frame.html
  47. +18 −3 doc/javadoc/overview-summary.html
  48. +15 −11 doc/javadoc/overview-tree.html
  49. +3 −0 doc/javadoc/package-list
  50. +6 −2 doc/javadoc/serialized-form.html
  51. +13 −13 doc/javadoc/stylesheet.css
  52. +3 −3 doc/parallelclients.html
  53. +3 −3 doc/tipsfaq.html
  54. +3 −3 doc/workload.html
  55. +6 −6 src/com/yahoo/ycsb/Client.java
  56. +410 −0 src/com/yahoo/ycsb/CommandLine.java
  57. +17 −5 src/com/yahoo/ycsb/DB.java
  58. +1 −1 src/com/yahoo/ycsb/measurements/OneMeasurement.java
View
@@ -2,4 +2,5 @@ build
.classpath
.project
.settings
-db/*/lib
+db/*/lib/*
+
View
@@ -44,6 +44,14 @@ doc/index.html file.
Revision History
================
+0.1.2 - 5/12/10
+
+- MongoDB binding (ypai)
+- Cassandra 0.7 binding (johanoskarsson)
+- Simple command line interface (brianfrankcooper)
+- Faster string generation (tlipcon)
+- Avoid Bytes conversion in HBaseClient (tlipcon)
+
0.1.1 - 4/25/10
- Compiles under 1.5
View
@@ -35,6 +35,11 @@
<antcall target="dbcompile"/>
</target>
+ <target name="dbcompile-mongodb" depends="compile">
+ <property name="db.dir" value="db/mongodb"/>
+ <antcall target="dbcompile"/>
+ </target>
+
<target name="compile">
<mkdir dir="${classes.dir}"/>
<javac srcdir="${src.dir}" destdir="${classes.dir}" classpathref="build.classpath" deprecation="on">
@@ -67,7 +72,12 @@
</target>
<target name="doc">
- <javadoc sourcepath="${src.dir}" destdir="${doc.dir}/javadoc" packagenames="com.yahoo.ycsb,com.yahoo.ycsb.workloads"/>
+ <javadoc destdir="${doc.dir}/javadoc" packagenames="com.yahoo.ycsb,com.yahoo.ycsb.workloads,com.yahoo.ycsb.db,com.yahoo.ycsb.generator,com.yahoo.ycsb.measurements">
+ <fileset dir="." defaultexcludes="yes">
+ <include name="src/**"/>
+ <include name="db/**/src/**"/>
+ </fileset>
+ </javadoc>
</target>
</project>
@@ -0,0 +1 @@
+This directory should contain jars for building and running Cassandra 0.5
@@ -36,8 +36,10 @@
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.cassandra.service.*;
+
+// XXXX if we do replication, fix the consistency levels
/**
- * XXXX if we do replication, fix the consistency levels
+ * Cassandra 0.5 client for YCSB framework
*/
public class CassandraClient5 extends DB
{
@@ -0,0 +1 @@
+This directory should contain jars for building and running Cassandra 0.6
@@ -33,8 +33,10 @@
import org.apache.cassandra.service.*;
import org.apache.cassandra.thrift.*;
+
+//XXXX if we do replication, fix the consistency levels
/**
- * XXXX if we do replication, fix the consistency levels
+ * Cassandra 0.6 client for YCSB framework
*/
public class CassandraClient6 extends DB
{
@@ -0,0 +1 @@
+This directory should contain jars for building and running Cassandra 0.7
@@ -35,8 +35,10 @@
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.cassandra.thrift.*;
+
+//XXXX if we do replication, fix the consistency levels
/**
- * XXXX if we do replication, fix the consistency levels
+ * Cassandra 0.7 client for YCSB framework
*/
public class CassandraClient7 extends DB
{
@@ -40,6 +40,9 @@
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.HBaseConfiguration;
+/**
+ * HBase client for YCSB framework
+ */
public class HBaseClient extends com.yahoo.ycsb.DB
{
public boolean _debug=false;
@@ -0,0 +1 @@
+This directory should contain jars for building and running MongoDB
@@ -0,0 +1,285 @@
+/**
+ * MongoDB client binding for YCSB.
+ *
+ * Submitted by Yen Pai on 5/11/2010.
+ *
+ * https://gist.github.com/000a66b8db2caf42467b#file_mongo_db.java
+ *
+ */
+
+package com.yahoo.ycsb.db;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Properties;
+import java.util.Set;
+import java.util.Vector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.mongodb.BasicDBObject;
+import com.mongodb.DBAddress;
+import com.mongodb.DBCollection;
+import com.mongodb.DBCursor;
+import com.mongodb.DBObject;
+import com.mongodb.Mongo;
+import com.mongodb.DB.WriteConcern;
+import com.yahoo.ycsb.DB;
+import com.yahoo.ycsb.DBException;
+
+/**
+* MongoDB client for YCSB framework.
+*
+* Properties to set:
+*
+* mongodb.url=mongodb://localhost:27017
+* mongodb.database=ycsb
+*
+* @author ypai
+*
+*/
+public class MongoDbClient extends DB {
+
+ private static final Logger logger = LoggerFactory.getLogger(MongoDbClient.class);
+
+ private Mongo mongo;
+ private WriteConcern writeConcern;
+ private String database;
+
+ /**
+* Initialize any state for this DB. Called once per DB instance; there is
+* one DB instance per client thread.
+*/
+ public void init() throws DBException {
+ // initialize MongoDb driver
+ Properties props = getProperties();
+ String url = props.getProperty("mongodb.url");
+ database = props.getProperty("mongodb.database");
+ String writeConcernType = props.getProperty("mongodb.writeConcern");
+
+ if ("none".equals(writeConcernType)) {
+ writeConcern = WriteConcern.NONE;
+ } else if ("strict".equals(writeConcernType)) {
+ writeConcern = WriteConcern.STRICT;
+ } else if ("normal".equals(writeConcernType)) {
+ writeConcern = WriteConcern.NORMAL;
+ }
+
+ try {
+ // strip out prefix since Java driver doesn't currently support
+ // standard
+ // connection format URL yet
+ // http://www.mongodb.org/display/DOCS/Connections
+ if (url.startsWith("mongodb://")) {
+ url = url.substring(10);
+ }
+
+ mongo = new Mongo(new DBAddress(url));
+ } catch (Exception e1) {
+ logger.error(
+ "Could not initialize MongoDB connection pool for Loader: "
+ + e1, e1);
+ return;
+ }
+
+ }
+
+ @Override
+ /**
+* Delete a record from the database.
+*
+* @param table The name of the table
+* @param key The record key of the record to delete.
+* @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes.
+*/
+ public int delete(String table, String key) {
+ try {
+ com.mongodb.DB db = mongo.getDB(database);
+ db.requestStart();
+ DBCollection collection = db.getCollection(table);
+ DBObject q = new BasicDBObject().append("_id", key);
+ if (writeConcern.equals(WriteConcern.STRICT)) {
+ q.put("$atomic", true);
+ }
+ collection.remove(q);
+
+ // see if record was deleted
+ DBObject errors = db.getLastError();
+
+ db.requestDone();
+
+ return (Long) errors.get("n") == 1 ? 0 : 1;
+ } catch (Exception e) {
+ logger.error(e + "", e);
+ return 1;
+ }
+
+ }
+
+ @Override
+ /**
+* Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified
+* record key.
+*
+* @param table The name of the table
+* @param key The record key of the record to insert.
+* @param values A HashMap of field/value pairs to insert in the record
+* @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes.
+*/
+ public int insert(String table, String key, HashMap<String, String> values) {
+ try {
+ com.mongodb.DB db = mongo.getDB(database);
+ db.requestStart();
+ DBCollection collection = db.getCollection(table);
+ DBObject r = new BasicDBObject().append("_id", key);
+ r.putAll(values);
+
+ collection.setWriteConcern(writeConcern);
+
+ collection.insert(r);
+
+ // determine if record was inserted
+ DBObject errors = db.getLastError();
+
+ db.requestDone();
+
+ return (Long) errors.get("n") == 1 ? 0 : 1;
+ } catch (Exception e) {
+ logger.error(e + "", e);
+ return 1;
+ }
+
+ }
+
+ @Override
+ @SuppressWarnings ("unchecked")
+ /**
+* Read a record from the database. Each field/value pair from the result will be stored in a HashMap.
+*
+* @param table The name of the table
+* @param key The record key of the record to read.
+* @param fields The list of fields to read, or null for all of them
+* @param result A HashMap of field/value pairs for the result
+* @return Zero on success, a non-zero error code on error or "not found".
+*/
+ public int read(String table, String key, Set<String> fields,
+ HashMap<String, String> result) {
+ try {
+ com.mongodb.DB db = mongo.getDB(database);
+ db.requestStart();
+ DBCollection collection = db.getCollection(table);
+ DBObject q = new BasicDBObject().append("_id", key);
+ DBObject fieldsToReturn = new BasicDBObject();
+ boolean returnAllFields = fields == null;
+
+ DBObject queryResult = null;
+ if (!returnAllFields) {
+ Iterator<String> iter = fields.iterator();
+ while (iter.hasNext()) {
+ fieldsToReturn.put(iter.next(), 1);
+ }
+ queryResult = collection.findOne(q, fieldsToReturn);
+ } else {
+ queryResult = collection.findOne(q);
+ }
+
+ if (queryResult != null) {
+ //toMap() returns a Map, but result.putAll() expects a Map<String,String>. Hence, the suppress warnings.
+ result.putAll(queryResult.toMap());
+ }
+
+ db.requestDone();
+
+ return queryResult != null ? 0 : 1;
+ } catch (Exception e) {
+ logger.error(e + "", e);
+ return 1;
+ }
+
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ /**
+* Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in a HashMap.
+*
+* @param table The name of the table
+* @param startkey The record key of the first record to read.
+* @param recordcount The number of records to read
+* @param fields The list of fields to read, or null for all of them
+* @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record
+* @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes.
+*/
+ public int scan(String table, String startkey, int recordcount,
+ Set<String> fields, Vector<HashMap<String, String>> result) {
+ try {
+ com.mongodb.DB db = mongo.getDB(database);
+ db.requestStart();
+ DBCollection collection = db.getCollection(table);
+ // { "_id":{"$gte":startKey, "$lte":{"appId":key+"\uFFFF"}} }
+ DBObject scanRange = new BasicDBObject().append("$gte", startkey);
+ DBObject q = new BasicDBObject().append("_id", scanRange);
+ DBCursor cursor = collection.find(q).limit(recordcount);
+ while (cursor.hasNext()) {
+ //toMap() returns a Map, but result.add() expects a Map<String,String>. Hence, the suppress warnings.
+ result.add((HashMap<String, String>) cursor.next().toMap());
+ }
+
+ db.requestDone();
+
+ return 0;
+ } catch (Exception e) {
+ logger.error(e + "", e);
+ return 1;
+ }
+
+ }
+
+ @Override
+ /**
+* Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified
+* record key, overwriting any existing values with the same field name.
+*
+* @param table The name of the table
+* @param key The record key of the record to write.
+* @param values A HashMap of field/value pairs to update in the record
+* @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes.
+*/
+ public int update(String table, String key, HashMap<String, String> values) {
+ try {
+ com.mongodb.DB db = mongo.getDB(database);
+ db.requestStart();
+ DBCollection collection = db.getCollection(table);
+ DBObject q = new BasicDBObject().append("_id", key);
+ DBObject u = new BasicDBObject();
+ DBObject fieldsToSet = new BasicDBObject();
+ Iterator<String> keys = values.keySet().iterator();
+ String tmpKey = null, tmpVal = null;
+ while (keys.hasNext()) {
+ tmpKey = keys.next();
+ tmpVal = values.get(tmpKey);
+ fieldsToSet.put(tmpKey, tmpVal);
+
+ }
+ u.put("$set", fieldsToSet);
+
+ collection.setWriteConcern(writeConcern);
+
+ collection.update(q, u);
+
+ // determine if record was inserted
+ DBObject errors = db.getLastError();
+
+ db.requestDone();
+
+ return (Long) errors.get("n") == 1 ? 0 : 1;
+ } catch (Exception e) {
+ logger.error(e + "", e);
+ return 1;
+ }
+
+ }
+}
+
+
@@ -3,8 +3,8 @@
<TITLE>YCSB - Core workload package properties</TITLE>
</HEAD>
<BODY>
-<H1><img src="images/ycsblogo-small.png"> Yahoo! Cloud Serving Benchmark</H1>
-<H3>Version 0.1</H3>
+<H1><img src="images/ycsb.jpg" width=150> Yahoo! Cloud Serving Benchmark</H1>
+<H3>Version 0.1.2</H3>
<HR>
<A HREF="index.html">Home</A> - <A href="coreworkloads.html">Core workloads</A> - <a href="tipsfaq.html">Tips and FAQ</A>
<HR>
Oops, something went wrong.

0 comments on commit ee88bbf

Please sign in to comment.