GORA-437 implement couchdb datastore

,added a new store for gora as gora-couchdb ,added unit tests ,added javadocs ,bugfixed
apache · Aug 13, 2016 · aad5878 · aad5878
1 parent ed80810
commit aad5878
Show file tree

Hide file tree

Showing 21 changed files with 1,626 additions and 2 deletions.
diff --git a/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java b/gora-accumulo/src/main/java/org/apache/gora/accumulo/store/AccumuloStore.java
@@ -690,6 +690,7 @@ else if (effectiveSchema.getType() == Type.MAP) {
           }
           // continue like a regular top-level union
         case RECORD:
+          //TODO duplicated code @see org.apache.gora.util.IOUtils#serialize()
           SpecificDatumWriter<Object> writer = new SpecificDatumWriter<>(field.schema());
           ByteArrayOutputStream os = new ByteArrayOutputStream();
           org.apache.avro.io.BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(os, null);

diff --git a/gora-cassandra/src/main/java/org/apache/gora/cassandra/store/CassandraStore.java b/gora-cassandra/src/main/java/org/apache/gora/cassandra/store/CassandraStore.java
@@ -511,6 +511,7 @@ private void addOrUpdateField(K key, Field field, Schema schema, Object value) {
           if (value instanceof PersistentBase) {
             PersistentBase persistentBase = (PersistentBase) value;            
             try {
+              //TODO duplicated code @see org.apache.gora.util.IOUtils#serialize()
               byte[] byteValue = AvroSerializerUtil.serializer(persistentBase, schema);
               this.cassandraClient.addColumn(key, field.name(), byteValue);
             } catch (IOException e) {

diff --git a/gora-core/src/test/java/org/apache/gora/GoraTestDriver.java b/gora-core/src/test/java/org/apache/gora/GoraTestDriver.java
@@ -94,14 +94,18 @@ public void tearDown() throws Exception {
   }
 
   protected void setProperties(Properties properties) {
+    //FIXME This method contains no code.
   }
 
   @SuppressWarnings("unchecked")
   public<K, T extends Persistent> DataStore<K,T>
     createDataStore(Class<K> keyClass, Class<T> persistentClass) throws GoraException {
-    setProperties(DataStoreFactory.createProps());
+
+    Properties properties = DataStoreFactory.createProps();
+
     DataStore<K,T> dataStore = DataStoreFactory.createDataStore(
-        (Class<? extends DataStore<K,T>>)dataStoreClass, keyClass, persistentClass, conf);
+        (Class<? extends DataStore<K,T>>)dataStoreClass, keyClass, persistentClass, conf, properties);
+
     dataStores.add(dataStore);
 
     log.info("Datastore for "+persistentClass+" was added.");

diff --git a/gora-couchdb/pom.xml b/gora-couchdb/pom.xml
@@ -0,0 +1,209 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.apache.gora</groupId>
+    <artifactId>gora</artifactId>
+    <version>0.7-SNAPSHOT</version>
+    <relativePath>../</relativePath>
+  </parent>
+  <artifactId>gora-couchdb</artifactId>
+  <packaging>bundle</packaging>
+
+  <name>Apache Gora :: CouchDB</name>
+  <url>http://gora.apache.org</url>
+  <description>The Apache Gora open source framework provides an in-memory data model and 
+    persistence for big data. Gora supports persisting to column stores, key value stores, 
+    document stores and RDBMSs, and analyzing the data with extensive Apache Hadoop MapReduce 
+    support.</description>
+  <inceptionYear>2010</inceptionYear>
+  <organization>
+    <name>The Apache Software Foundation</name>
+    <url>http://www.apache.org/</url>
+  </organization>
+  <issueManagement>
+    <system>JIRA</system>
+    <url>https://issues.apache.org/jira/browse/GORA</url>
+  </issueManagement>
+  <ciManagement>
+    <system>Jenkins</system>
+    <url>https://builds.apache.org/job/Gora-trunk/</url>
+  </ciManagement>
+
+  <properties>
+    <osgi.import>*</osgi.import>
+    <osgi.export>org.apache.gora.hbase*;version="${project.version}";-noimport:=true</osgi.export>
+  </properties>
+
+  <build>
+    <directory>target</directory>
+    <outputDirectory>target/classes</outputDirectory>
+    <finalName>${project.artifactId}-${project.version}</finalName>
+    <testOutputDirectory>target/test-classes</testOutputDirectory>
+    <testSourceDirectory>src/test/java</testSourceDirectory>
+    <sourceDirectory>src/main/java</sourceDirectory>
+    <testResources>
+      <testResource>
+        <directory>${project.basedir}/src/test/conf</directory>
+        <includes>
+          <include>**/*</include>
+        </includes>
+        <!--targetPath>${project.basedir}/target/classes/</targetPath -->
+      </testResource>
+    </testResources>
+    <plugins>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>build-helper-maven-plugin</artifactId>
+        <version>${build-helper-maven-plugin.version}</version>
+        <executions>
+          <execution>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>add-source</goal>
+            </goals>
+            <configuration>
+              <sources>
+                <source>src/examples/java</source>
+              </sources>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+  <repositories>
+    <repository>
+      <id>clojars.org</id>
+      <url>http://clojars.org/repo</url>
+    </repository>
+  </repositories>
+
+  <dependencies>
+    <!-- Gora Internal Dependencies -->
+    <dependency>
+      <groupId>org.apache.gora</groupId>
+      <artifactId>gora-core</artifactId>
+      <scope>compile</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.gora</groupId>
+      <artifactId>gora-core</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+
+
+    <dependency>
+      <groupId>org.apache.gora</groupId>
+      <artifactId>gora-core</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+
+    <!-- END of Gora Internal Dependencies -->
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client</artifactId>
+      <scope>compile</scope>
+      <optional>true</optional>
+    </dependency>
+
+    <!-- Apache CouchDB java client -->
+    <dependency>
+      <groupId>org.ektorp</groupId>
+      <artifactId>org.ektorp</artifactId>
+      <version>1.4.2</version>
+    </dependency>
+
+
+    <dependency>
+      <groupId>org.apache.avro</groupId>
+      <artifactId>avro</artifactId>
+      <scope>compile</scope>
+    </dependency>
+
+    <!-- Misc Dependencies -->
+    <dependency>
+      <groupId>org.jdom</groupId>
+      <artifactId>jdom</artifactId>
+      <scope>compile</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <scope>compile</scope>
+    </dependency>
+
+    <!-- Logging Dependencies -->
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <scope>compile</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-simple</artifactId>
+      <scope>compile</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>log4j</groupId>
+      <artifactId>log4j</artifactId>
+      <scope>runtime</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>javax.jms</groupId>
+          <artifactId>jms</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <dependency>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>testcontainers</artifactId>
+      <version>1.1.0</version>
+    </dependency>
+    <!-- END of Logging Dependencies -->
+
+    <!-- Testing Dependencies -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-minicluster</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <!-- END of Testing Dependencies -->
+  </dependencies>
+
+</project>
diff --git a/gora-couchdb/src/main/java/org/apache/gora/couchdb/query/CouchDBQuery.java b/gora-couchdb/src/main/java/org/apache/gora/couchdb/query/CouchDBQuery.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.couchdb.query;
+
+import org.apache.gora.couchdb.store.CouchDBStore;
+import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.impl.QueryBase;
+import org.apache.gora.store.DataStore;
+
+/**
+ * CouchDB specific implementation of the {@link Query} interface.
+ */
+public class CouchDBQuery<K, T extends PersistentBase> extends QueryBase<K, T> {
+
+  final CouchDBStore store;
+
+  public CouchDBQuery() {
+    super(null);
+    store = null;
+  }
+
+  public CouchDBQuery(DataStore<K, T> dataStore) {
+    super(dataStore);
+    store = (CouchDBStore) dataStore;
+  }
+}
diff --git a/gora-couchdb/src/main/java/org/apache/gora/couchdb/query/CouchDBResult.java b/gora-couchdb/src/main/java/org/apache/gora/couchdb/query/CouchDBResult.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.couchdb.query;
+
+import org.apache.gora.couchdb.store.CouchDBStore;
+import org.apache.gora.persistency.Persistent;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.impl.ResultBase;
+import org.apache.gora.store.DataStore;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * CouchDB specific implementation of the {@link org.apache.gora.query.Result}
+ * interface.
+ */
+public class CouchDBResult<K, T extends Persistent> extends ResultBase<K, T> {
+
+  /**
+   * Result set containing query results
+   */
+  private List<Map> result;
+
+  protected CouchDBStore dataStore;
+  int position = 0;
+
+  /**
+   * Constructor for the result set
+   *
+   * @param dataStore Data store used
+   * @param query     Query used
+   * @param result    Result obtained from querying
+   */
+  public CouchDBResult(DataStore<K, T> dataStore, Query<K, T> query, List<Map> result) {
+    super(dataStore, query);
+    this.result = result;
+    this.dataStore = (CouchDBStore) dataStore;
+  }
+
+  /**
+   * Gets the next item
+   */
+  @Override
+  protected boolean nextInner() throws IOException {
+    if (result == null || result.size() <= 0 || position >= result.size()) {
+      return false;
+    }
+    key = (K) result.get(position).get("_id");
+    persistent = (T) dataStore.newInstance(result.get(position++), query.getFields());
+    return persistent != null;
+  }
+
+  /**
+   * Gets the items reading progress
+   */
+  @Override
+  public float getProgress() throws IOException, InterruptedException {
+    if (result != null && result.size() > 0) {
+      return (float) position / (float) result.size();
+    } else {
+      return 0;
+    }
+  }
+
+  public List<Map> getResultData() {
+    return result;
+  }
+}