Merge pull request #1 from ivankelly/concurrent-read-v2

Concurrent read v2
apache · Dec 4, 2017 · 73473f6 · 73473f6
2 parents 7aa76f7 + cd3599c
commit 73473f6
Show file tree

Hide file tree

Showing 291 changed files with 8,877 additions and 4,408 deletions.
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
@@ -1,4 +1,4 @@
-Is this a question, feature request, or bug report?
+Is this a question, feature request, bug report, or bookkeeper proposal?
 
 **QUESTION**
 
@@ -29,3 +29,10 @@ Have you checked our documentation at http://bookkeeper.apache.org/ , If you cou
 
 - What did you see instead?
 
+**BP**
+
+This is the master ticket for tracking BP-&lt;bp-number&gt; :
+
+[a short description for this BP]
+
+Proposal PR - #&lt;pr-number&gt;
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -5,6 +5,15 @@ Descriptions of the changes in this PR:
 > ---
 > Be sure to do all of the following to help us incorporate your contribution
 > quickly and easily:
+>
+> If this PR is a BookKeeper Proposal (BP):
+>
+> - [ ] Make sure the PR title is formatted like:
+>     `<BP-#>: Description of bookkeeper proposal`
+>     `e.g. BP-1: 64 bits ledger is support`
+> - [ ] Attach the master issue link in the description of this PR.
+>
+> Otherwise:
 > 
 > - [ ] Make sure the PR title is formatted like:
 >     `<Issue # or BOOKKEEPER-#>: Description of pull request`

diff --git a/bookkeeper-benchmark/pom.xml b/bookkeeper-benchmark/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <artifactId>bookkeeper</artifactId>
     <groupId>org.apache.bookkeeper</groupId>
-    <version>4.6.0-SNAPSHOT</version>
+    <version>4.7.0-SNAPSHOT</version>
   </parent>
   <groupId>org.apache.bookkeeper</groupId>
   <artifactId>bookkeeper-benchmark</artifactId>
@@ -31,13 +31,6 @@
   </properties>
   <build>
     <plugins>
-      <plugin>
-        <artifactId>maven-assembly-plugin</artifactId>
-        <version>${maven-assembly-plugin.version}</version>
-        <configuration>
-	  <skipAssembly>true</skipAssembly>
-        </configuration>
-      </plugin>
       <plugin>
       	<groupId>org.apache.maven.plugins</groupId>
 	<artifactId>maven-surefire-plugin</artifactId>
@@ -83,24 +76,6 @@
     </plugins>
   </build>
   <dependencies>
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-log4j12</artifactId>
-      <version>${slf4j.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.zookeeper</groupId>
-      <artifactId>zookeeper</artifactId>
-      <version>${zookeeper.version}</version>
-      <type>jar</type>
-      <scope>compile</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>net.java.dev.javacc</groupId>
-          <artifactId>javacc</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
     <dependency>
       <groupId>org.apache.zookeeper</groupId>
       <artifactId>zookeeper</artifactId>
@@ -126,17 +101,6 @@
         </exclusion>
       </exclusions>
     </dependency>
-    <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty-all</artifactId>
-      <version>${netty.version}</version>
-      <scope>compile</scope>
-    </dependency>
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-      <version>${guava.version}</version>
-    </dependency>
     <dependency>
       <groupId>org.apache.bookkeeper</groupId>
       <artifactId>bookkeeper-server</artifactId>
@@ -151,25 +115,5 @@
       <scope>test</scope>
       <type>test-jar</type>
     </dependency>
-    <dependency>
-       <groupId>org.apache.hadoop</groupId>
-       <artifactId>hadoop-common</artifactId>
-       <version>0.23.1</version>
-       <scope>compile</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs</artifactId>
-      <version>0.23.1</version>
-      <scope>compile</scope>
-      <!-- commons-daemon has a malformed pom, which can cause the build to fail in an
-           environment which uses an artifactory cache. -->
-      <exclusions>
-        <exclusion>
-          <groupId>commons-daemon</groupId>
-          <artifactId>commons-daemon</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
   </dependencies>
 </project>
diff --git a/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/MySqlClient.java b/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/MySqlClient.java
diff --git a/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/TestClient.java b/bookkeeper-benchmark/src/main/java/org/apache/bookkeeper/benchmark/TestClient.java
@@ -48,10 +48,6 @@
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.cli.PosixParser;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -140,19 +136,6 @@ public void run() {
                 for (int i = 0; i < numFiles; i++) {
                     clients.add(new BKClient(handles, data, runfor, cmd.hasOption("sync")));
                 }
-            } else if (target.equals("hdfs")) {
-                FileSystem fs = FileSystem.get(new Configuration());
-                LOG.info("Default replication for HDFS: {}", fs.getDefaultReplication());
-
-                List<FSDataOutputStream> streams = new ArrayList<FSDataOutputStream>();
-                for (int i = 0; i < numFiles; i++) {
-                    String path = cmd.getOptionValue("path", "/foobar");
-                    streams.add(fs.create(new Path(path + runid + "_" + i)));
-                }
-
-                for (int i = 0; i < numThreads; i++) {
-                    clients.add(new HDFSClient(streams, data, runfor));
-                }
             } else if (target.equals("fs")) {
                 List<FileOutputStream> streams = new ArrayList<FileOutputStream>();
                 for (int i = 0; i < numFiles; i++) {
@@ -211,45 +194,6 @@ public void run() {
         timeouter.cancel();
     }
 
-    static class HDFSClient implements Callable<Long> {
-        final List<FSDataOutputStream> streams;
-        final byte[] data;
-        final long time;
-        final Random r;
-
-        HDFSClient(List<FSDataOutputStream> streams, byte[] data, long time) {
-            this.streams = streams;
-            this.data = data;
-            this.time = time;
-            this.r = new Random(System.identityHashCode(this));
-        }
-
-        public Long call() {
-            try {
-                long count = 0;
-                long start = System.currentTimeMillis();
-                long stopat = start + time;
-                while (System.currentTimeMillis() < stopat) {
-                    FSDataOutputStream stream = streams.get(r.nextInt(streams.size()));
-                    synchronized (stream) {
-                        stream.write(data);
-                        stream.flush();
-                        stream.hflush();
-                    }
-                    count++;
-                }
-
-                long time = (System.currentTimeMillis() - start);
-                LOG.info("Worker finished processing writes (ms): {} TPT: {} op/s", time,
-                         count / ((double) time / 1000));
-                return count;
-            } catch (IOException ioe) {
-                LOG.error("Exception in worker thread", ioe);
-                return 0L;
-            }
-        }
-    }
-
     static class FileClient implements Callable<Long> {
         final List<FileOutputStream> streams;
         final byte[] data;

diff --git a/bookkeeper-common/pom.xml b/bookkeeper-common/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.bookkeeper</groupId>
     <artifactId>bookkeeper</artifactId>
-    <version>4.6.0-SNAPSHOT</version>
+    <version>4.7.0-SNAPSHOT</version>
   </parent>
   <artifactId>bookkeeper-common</artifactId>
   <name>Apache BookKeeper :: Common</name>
@@ -47,6 +47,11 @@
       <version>${commons-lang3.version}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.google.errorprone</groupId>
+      <artifactId>error_prone_annotations</artifactId>
+      <version>2.1.2</version>
+    </dependency>
   </dependencies>
   <build>
     <plugins>