Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

initial version of mongodb

  • Loading branch information...
commit bc1bbd00bb78ffde99cf127ce21248261467f994 1 parent a7039d6
Sam Pullara authored
58 mongodb/pom.xml
View
@@ -0,0 +1,58 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>all</artifactId>
+ <groupId>com.github.spullara.avrobase</groupId>
+ <version>0.2-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>mongodb</artifactId>
+ <packaging>jar</packaging>
+
+ <name>mongodb</name>
+ <url>http://maven.apache.org</url>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.github.spullara.avrobase</groupId>
+ <artifactId>base</artifactId>
+ <version>0.2-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mongodb</groupId>
+ <artifactId>mongo-java-driver</artifactId>
+ <version>2.6.2</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-maven-plugin</artifactId>
+ <version>${avroplugin.version}</version>
+ <executions>
+ <execution>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>schema</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
166 mongodb/src/main/java/avrobase/mongodb/MongoAB.java
View
@@ -0,0 +1,166 @@
+package avrobase.mongodb;
+
+import avrobase.AvroBaseException;
+import avrobase.AvroBaseImpl;
+import avrobase.AvroFormat;
+import avrobase.Row;
+import com.mongodb.BasicDBObject;
+import com.mongodb.DB;
+import com.mongodb.DBCollection;
+import com.mongodb.DBCursor;
+import com.mongodb.DBObject;
+import org.apache.avro.Schema;
+import org.apache.avro.specific.SpecificData;
+import org.apache.avro.specific.SpecificRecord;
+import org.apache.avro.util.Utf8;
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+
+/**
+ * TODO: Edit this
+ * <p/>
+ * User: sam
+ * Date: 6/4/11
+ * Time: 6:54 PM
+ */
+public class MongoAB<T extends SpecificRecord, K> extends AvroBaseImpl<T, K> {
+
+ private final Schema readerSchema;
+ private final DBCollection rows;
+
+ public MongoAB(DB db, String typeName, Schema readerSchema) {
+ super(readerSchema, AvroFormat.JSON);
+ this.readerSchema = readerSchema;
+ this.rows = db.getCollection(typeName);
+ rows.createIndex(b("id", 1));
+ }
+
+ private BasicDBObject b() {
+ return new BasicDBObject();
+ }
+
+ private BasicDBObject b(String name, Object value) {
+ return new BasicDBObject(name, value);
+ }
+
+ @Override
+ public Row<T, K> get(K row) throws AvroBaseException {
+ DBCursor cursor = rows.find(r(row));
+ if (cursor.hasNext()) {
+ DBObject ro = cursor.next();
+ return newrow(row, ro);
+ }
+ return null;
+ }
+
+ private Row<T, K> newrow(K row, DBObject ro) {
+ T ao = getAvroObject(ro);
+ return new Row<T, K>(ao, row, (Long) ro.get("version"));
+ }
+
+ private T getAvroObject(DBObject ro) {
+ BasicDBObject vo = (BasicDBObject) ro.get("value");
+ Class c = SpecificData.get().getClass(readerSchema);
+ T ao;
+ try {
+ ao = (T) c.newInstance();
+ for (Schema.Field field : readerSchema.getFields()) {
+ String name = field.name();
+ Object v = vo.get(name);
+ if (v instanceof byte[]) {
+ v = ByteBuffer.wrap((byte[]) v);
+ }
+ ao.put(field.pos(), v);
+ }
+ } catch (Exception e) {
+ throw new AvroBaseException("Could not create object", e);
+ }
+ return ao;
+ }
+
+ @Override
+ public K create(T value) throws AvroBaseException {
+ return null;
+ }
+
+ @Override
+ public void put(K row, T value) throws AvroBaseException {
+ BasicDBObject ro = r(row);
+ ro.put("version", 1L);
+ ro.put("value", getDBObject(value));
+ rows.update(r(row), ro, true, false);
+ }
+
+ private BasicDBObject getDBObject(T value) {
+ BasicDBObject vo = new BasicDBObject();
+ for (Schema.Field field : actualSchema.getFields()) {
+ int pos = field.pos();
+ Object val = value.get(pos);
+ if (val instanceof Utf8) {
+ val = val.toString();
+ } else if (val instanceof ByteBuffer) {
+ val = ((ByteBuffer)val).array();
+ }
+ vo.put(field.name(), val);
+ }
+ return vo;
+ }
+
+ @Override
+ public boolean put(K row, T value, long version) throws AvroBaseException {
+ BasicDBObject ro = b("$set", r(row));
+ ro.put("$set", b("value", getDBObject(value)));
+ ro.put("$inc", b("version", 1));
+ BasicDBObject query = r(row);
+ query.put("version", version);
+ return rows.update(query, ro, false, false).getN() == 1;
+ }
+
+ @Override
+ public void delete(K row) throws AvroBaseException {
+ rows.findAndRemove(r(row));
+ }
+
+ private BasicDBObject r(K row) {
+ return new BasicDBObject("id", row);
+ }
+
+ @Override
+ public Iterable<Row<T, K>> scan(K startRow, K stopRow) throws AvroBaseException {
+ BasicDBObject b = b();
+ if (startRow != null) {
+ if (stopRow == null) {
+ b.put("id", b("$gte", startRow));
+ } else {
+ BasicDBObject query = b("$gte", startRow);
+ query.put("$lt", stopRow);
+ b = b("id", query);
+ }
+ } else if (stopRow != null) b.put("id", b("$lt", stopRow));
+ final DBCursor dbCursor = rows.find(b);
+ dbCursor.sort(b("id", 1));
+ return new Iterable<Row<T, K>>() {
+ @Override
+ public Iterator<Row<T, K>> iterator() {
+ final Iterator<DBObject> iterator = dbCursor.iterator();
+ return new Iterator<Row<T, K>>() {
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public Row<T, K> next() {
+ DBObject next = iterator.next();
+ return newrow((K) next.get("id"), next);
+ }
+
+ @Override
+ public void remove() {
+ }
+ };
+ }
+ };
+ }
+}
61 mongodb/src/test/avro/User.avsc
View
@@ -0,0 +1,61 @@
+{
+ "type":"record",
+ "name":"User",
+ "namespace":"bagcheck",
+ "fields": [
+ {
+ "name":"firstName",
+ "type":"string",
+ "default": ""
+ },
+ {
+ "name":"lastName",
+ "type":"string",
+ "default": ""
+ },
+ {
+ "name":"email",
+ "type":"string",
+ "default": ""
+ },
+ {
+ "name":"birthday",
+ "type":["string", "null"],
+ "default": ""
+ },
+ {
+ "name":"gender",
+ "type":[{"type":"enum", "name":"GenderType", "symbols":["FEMALE", "MALE"]}, "null"]
+ },
+ {
+ "name":"image",
+ "type":"string",
+ "default": ""
+ },
+ {
+ "name":"title",
+ "type":["string", "null"],
+ "default": ""
+ },
+ {
+ "name":"description",
+ "type":["string", "null"],
+ "default": ""
+ },
+ {
+ "name":"location",
+ "type":["string", "null"],
+ "default": ""
+ },
+ {
+ "name":"password",
+ "type":"bytes",
+ "default": ""
+ },
+ {
+ "name":"mobile",
+ "type":["string", "null"],
+ "default": "null"
+ }
+ ]
+}
133 mongodb/src/test/java/avrobase/mongodb/MongoABTest.java
View
@@ -0,0 +1,133 @@
+package avrobase.mongodb;
+
+import avrobase.Mutator;
+import avrobase.Row;
+import bagcheck.User;
+import com.google.common.base.Charsets;
+import com.google.common.primitives.Ints;
+import com.mongodb.DB;
+import com.mongodb.Mongo;
+import org.apache.avro.util.Utf8;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+
+import static junit.framework.Assert.assertEquals;
+
+public class MongoABTest {
+
+ private static DB avrobasetest;
+ private byte[] row;
+
+ @BeforeClass
+ public static void setup() throws UnknownHostException {
+ Mongo mongo = new Mongo("localhost");
+ avrobasetest = mongo.getDB("avrobasetest");
+ }
+
+ @Test
+ public void putGet() throws InterruptedException {
+ MongoAB<User, byte[]> userRAB = getAB();
+ User user = getUser();
+ userRAB.put("test".getBytes(Charsets.UTF_8), user);
+ Thread.sleep(1000);
+ Row<User, byte[]> test = userRAB.get("test".getBytes(Charsets.UTF_8));
+ assertEquals(user, test.value);
+ }
+
+ @Test
+ public void putGet2() {
+ MongoAB<User, byte[]> userRAB = getAB();
+ User user = getUser();
+ row = "test".getBytes(Charsets.UTF_8);
+ userRAB.put(row, user);
+ userRAB.mutate(row, new Mutator<User>() {
+ @Override
+ public User mutate(User value) {
+ value.firstName = $("John");
+ return value;
+ }
+ });
+ Row<User, byte[]> test = userRAB.get("test".getBytes(Charsets.UTF_8));
+ user = getUser();
+ user.firstName = $("John");
+ assertEquals(user, test.value);
+ }
+
+ @Test
+ public void rdelete() {
+ MongoAB<User, byte[]> userRAB = getAB();
+ row = "test".getBytes(Charsets.UTF_8);
+ userRAB.delete(row);
+ Row<User, byte[]> test = userRAB.get("test".getBytes(Charsets.UTF_8));
+ assertEquals(null, test);
+ }
+
+ @Test
+ public void testScan() {
+ MongoAB<User, byte[]> userRAB = getAB();
+ User user = getUser();
+ long start;
+ start = System.currentTimeMillis();
+ for (int i = 0; i < 100000; i++) {
+ userRAB.put(Ints.toByteArray(i), user);
+ }
+ System.out.println(System.currentTimeMillis() - start);
+ start = System.currentTimeMillis();
+ int total;
+ start = System.currentTimeMillis();
+ total = 0;
+ for (Row<User, byte[]> userRow : userRAB.scan(Ints.toByteArray(50000), null)) {
+ total++;
+ }
+ assertEquals(50000, total);
+ System.out.println(System.currentTimeMillis() - start);
+ start = System.currentTimeMillis();
+ total = 0;
+ for (Row<User, byte[]> userRow : userRAB.scan(null, Ints.toByteArray(50000))) {
+ total++;
+ }
+ assertEquals(50000, total);
+ System.out.println(System.currentTimeMillis() - start);
+ start = System.currentTimeMillis();
+ total = 0;
+ for (Row<User, byte[]> userRow : userRAB.scan(Ints.toByteArray(25000), Ints.toByteArray(75000))) {
+ total++;
+ }
+ assertEquals(50000, total);
+ System.out.println(System.currentTimeMillis() - start);
+ total = 0;
+ for (Row<User, byte[]> userRow : userRAB.scan(null, null)) {
+ total++;
+ userRAB.delete(userRow.row);
+ }
+ assertEquals(100000, total);
+ System.out.println(System.currentTimeMillis() - start);
+ total = 0;
+ for (Row<User, byte[]> userRow : userRAB.scan(null, null)) {
+ total++;
+ }
+ assertEquals(0, total);
+ }
+
+ private User getUser() {
+ User user = new User();
+ user.email = $("spullara@yahoo.com");
+ user.firstName = $("Sam");
+ user.lastName = $("Pullara");
+ user.image = $("");
+ user.password = ByteBuffer.allocate(0);
+ return user;
+ }
+
+ Utf8 $(String s) {
+ return new Utf8(s);
+ }
+
+ private MongoAB<User, byte[]> getAB() {
+ return new MongoAB<User, byte[]>(avrobasetest, "users", User.SCHEMA$);
+ }
+
+}
1  pom.xml
View
@@ -21,6 +21,7 @@
<module>handlersocket</module>
<module>s3archive</module>
<module>caching</module>
+ <module>mongodb</module>
</modules>
<packaging>pom</packaging>
Please sign in to comment.
Something went wrong with that request. Please try again.