[bookkeeper] bump bookkeeper version to 4.9.0 (#2906)

*Motivation*

This change was part of #2714. Moving it out as a separate change to make reviews easier.

*Changes*

Bump bookkeeper version to 4.9.0

distribution/server/src/assemble/LICENSE.bin.txt

@@ -323,7 +323,7 @@ The Apache Software License, Version 2.0
      - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.9.7.jar
      - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.9.7.jar
  * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.6.2.jar
- * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-1.0.0.jar
+ * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-1.12.0.jar
  * Gson -- com.google.code.gson-gson-2.8.2.jar
  * Guava -- com.google.guava-guava-21.0.jar
  * Netty Reactive Streams -- com.typesafe.netty-netty-reactive-streams-2.0.0.jar
@@ -350,8 +350,8 @@ The Apache Software License, Version 2.0
     - org.apache.commons-commons-lang3-3.4.jar
  * Netty
     - io.netty-netty-3.10.1.Final.jar
-    - io.netty-netty-all-4.1.22.Final.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.7.Final.jar
+    - io.netty-netty-all-4.1.32.Final.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.20.Final.jar
  * Prometheus client
     - io.prometheus-simpleclient-0.5.0.jar
     - io.prometheus-simpleclient_common-0.5.0.jar
@@ -368,29 +368,32 @@ The Apache Software License, Version 2.0
     - org.apache.logging.log4j-log4j-web-2.10.0.jar
  * Java Native Access JNA -- net.java.dev.jna-jna-4.2.0.jar
  * BookKeeper
-    - org.apache.bookkeeper-bookkeeper-common-4.7.3.jar
-    - org.apache.bookkeeper-bookkeeper-proto-4.7.3.jar
-    - org.apache.bookkeeper-bookkeeper-server-4.7.3.jar
-    - org.apache.bookkeeper-circe-checksum-4.7.3.jar
-    - org.apache.bookkeeper-statelib-4.7.3.jar
-    - org.apache.bookkeeper-stream-storage-api-4.7.3.jar
-    - org.apache.bookkeeper-stream-storage-common-4.7.3.jar
-    - org.apache.bookkeeper-stream-storage-java-client-4.7.3.jar
-    - org.apache.bookkeeper-stream-storage-java-client-base-4.7.3.jar
-    - org.apache.bookkeeper-stream-storage-proto-4.7.3.jar
-    - org.apache.bookkeeper-stream-storage-server-4.7.3.jar
-    - org.apache.bookkeeper-stream-storage-service-api-4.7.3.jar
-    - org.apache.bookkeeper-stream-storage-service-impl-4.7.3.jar
-    - org.apache.bookkeeper.http-http-server-4.7.3.jar
-    - org.apache.bookkeeper.http-vertx-http-server-4.7.3.jar
-    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.7.3.jar
-    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.7.3.jar
-    - org.apache.bookkeeper.tests-stream-storage-tests-common-4.7.3.jar
-    - org.apache.distributedlog-distributedlog-common-4.7.3.jar
-    - org.apache.distributedlog-distributedlog-core-4.7.3-tests.jar
-    - org.apache.distributedlog-distributedlog-core-4.7.3.jar
-    - org.apache.distributedlog-distributedlog-protocol-4.7.3.jar
-    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.7.3.jar
+    - org.apache.bookkeeper-bookkeeper-common-4.9.0.jar
+    - org.apache.bookkeeper-bookkeeper-common-allocator-4.9.0.jar
+    - org.apache.bookkeeper-bookkeeper-proto-4.9.0.jar
+    - org.apache.bookkeeper-bookkeeper-server-4.9.0.jar
+    - org.apache.bookkeeper-bookkeeper-tools-framework-4.9.0.jar
+    - org.apache.bookkeeper-circe-checksum-4.9.0.jar
+    - org.apache.bookkeeper-cpu-affinity-4.9.0.jar
+    - org.apache.bookkeeper-statelib-4.9.0.jar
+    - org.apache.bookkeeper-stream-storage-api-4.9.0.jar
+    - org.apache.bookkeeper-stream-storage-common-4.9.0.jar
+    - org.apache.bookkeeper-stream-storage-java-client-4.9.0.jar
+    - org.apache.bookkeeper-stream-storage-java-client-base-4.9.0.jar
+    - org.apache.bookkeeper-stream-storage-proto-4.9.0.jar
+    - org.apache.bookkeeper-stream-storage-server-4.9.0.jar
+    - org.apache.bookkeeper-stream-storage-service-api-4.9.0.jar
+    - org.apache.bookkeeper-stream-storage-service-impl-4.9.0.jar
+    - org.apache.bookkeeper.http-http-server-4.9.0.jar
+    - org.apache.bookkeeper.http-vertx-http-server-4.9.0.jar
+    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.9.0.jar
+    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.9.0.jar
+    - org.apache.bookkeeper.tests-stream-storage-tests-common-4.9.0.jar
+    - org.apache.distributedlog-distributedlog-common-4.9.0.jar
+    - org.apache.distributedlog-distributedlog-core-4.9.0-tests.jar
+    - org.apache.distributedlog-distributedlog-core-4.9.0.jar
+    - org.apache.distributedlog-distributedlog-protocol-4.9.0.jar
+    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.9.0.jar
  * LZ4 -- org.lz4-lz4-java-1.5.0.jar
  * AsyncHttpClient
     - org.asynchttpclient-async-http-client-2.7.0.jar
@@ -423,20 +426,20 @@ The Apache Software License, Version 2.0
  * Okio - com.squareup.okio-okio-1.13.0.jar
  * Javassist -- org.javassist-javassist-3.21.0-GA.jar
   * gRPC
-    - io.grpc-grpc-all-1.16.1.jar
-    - io.grpc-grpc-auth-1.16.1.jar
-    - io.grpc-grpc-context-1.16.1.jar
-    - io.grpc-grpc-core-1.16.1.jar
-    - io.grpc-grpc-netty-1.16.1.jar
-    - io.grpc-grpc-okhttp-1.16.1.jar
-    - io.grpc-grpc-protobuf-1.16.1.jar
-    - io.grpc-grpc-protobuf-lite-1.16.1.jar
-    - io.grpc-grpc-protobuf-nano-1.16.1.jar
-    - io.grpc-grpc-stub-1.16.1.jar
+    - io.grpc-grpc-all-1.18.0.jar
+    - io.grpc-grpc-auth-1.18.0.jar
+    - io.grpc-grpc-context-1.18.0.jar
+    - io.grpc-grpc-core-1.18.0.jar
+    - io.grpc-grpc-netty-1.18.0.jar
+    - io.grpc-grpc-okhttp-1.18.0.jar
+    - io.grpc-grpc-protobuf-1.18.0.jar
+    - io.grpc-grpc-protobuf-lite-1.18.0.jar
+    - io.grpc-grpc-protobuf-nano-1.18.0.jar
+    - io.grpc-grpc-stub-1.18.0.jar
     - io.grpc-grpc-testing-1.12.0.jar
   * OpenCensus
-    - io.opencensus-opencensus-api-0.12.3.jar
-    - io.opencensus-opencensus-contrib-grpc-metrics-0.12.3.jar
+    - io.opencensus-opencensus-api-0.18.0.jar
+    - io.opencensus-opencensus-contrib-grpc-metrics-0.18.0.jar
   * Paranamer
     - com.thoughtworks.paranamer-paranamer-2.7.jar
   * Jodah
@@ -477,6 +480,8 @@ The Apache Software License, Version 2.0
     - io.jsonwebtoken-jjwt-jackson-0.10.5.jar
   * JavaX Injection
     - javax.inject-javax.inject-1.jar
+  * JCTools - Java Concurrency Tools for the JVM
+    - org.jctools-jctools-core-2.1.2.jar
   * Vertx
     - io.vertx-vertx-auth-common-3.4.1.jar 
     - io.vertx-vertx-core-3.4.1.jar
@@ -501,6 +506,8 @@ MIT License
     - org.slf4j-jcl-over-slf4j-1.7.25.jar
  * Animal Sniffer Annotations
     - org.codehaus.mojo-animal-sniffer-annotations-1.17.jar
+ * The Checker Framework
+    - org.checkerframework-checker-compat-qual-2.5.2.jar 
 
 Protocol Buffers License
  * Protocol Buffers

distribution/server/src/assemble/NOTICE.bin.txt

@@ -200,3 +200,18 @@ http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html
     * Providing benchmark suite
   * Alec Wysoker
     * Performance and memory usage improvement
+
+------------------------------------------------------------------------------------
+- org.checkerframework-checker-compat-qual-2.5.2.jar
+
+A few parts of the Checker Framework have more permissive licenses.
+
+ * The annotations are licensed under the MIT License.  (The text of this
+   license appears below.)  More specifically, all the parts of the Checker
+   Framework that you might want to include with your own program use the
+   MIT License.  This is the checker-qual.jar file and all the files that
+   appear in it:  every file in a qual/ directory, plus utility files such
+   as NullnessUtil.java, RegexUtil.java, SignednessUtil.java, etc.
+   In addition, the cleanroom implementations of third-party annotations,
+   which the Checker Framework recognizes as aliases for its own
+   annotations, are licensed under the MIT License.

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpAddEntry.java

@@ -95,15 +95,10 @@ public void setCloseWhenDone(boolean closeWhenDone) {
 
     public void initiate() {
         ByteBuf duplicateBuffer = data.retainedDuplicate();
-        // duplicatedBuffer has refCnt=1 at this point
 
+        // internally asyncAddEntry() will take the ownership of the buffer and release it at the end
         lastInitTime = System.nanoTime();
         ledger.asyncAddEntry(duplicateBuffer, this, ctx);
-
-        // Internally, asyncAddEntry() is refCnt neutral to respect to the passed buffer and it will keep a ref on it
-        // until is done using it. We need to release this buffer here to balance the 1 refCnt added at the creation
-        // time.
-        duplicateBuffer.release();
     }
 
     public void failed(ManagedLedgerException e) {

managed-ledger/src/test/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java

@@ -33,11 +33,13 @@
 import org.apache.bookkeeper.client.AsyncCallback.CreateCallback;
 import org.apache.bookkeeper.client.AsyncCallback.DeleteCallback;
 import org.apache.bookkeeper.client.AsyncCallback.OpenCallback;
+import org.apache.bookkeeper.client.BKException.Code;
 import org.apache.bookkeeper.client.api.OpenBuilder;
 import org.apache.bookkeeper.client.api.ReadHandle;
 import org.apache.bookkeeper.client.impl.OpenBuilderBase;
 import org.apache.bookkeeper.common.concurrent.FutureUtils;
 import org.apache.bookkeeper.conf.ClientConfiguration;
+import org.apache.bookkeeper.net.BookieSocketAddress;
 import org.apache.zookeeper.ZooKeeper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -198,8 +200,9 @@ public OpenBuilder newOpenLedgerOp() {
             public CompletableFuture<ReadHandle> execute() {
                 return getProgrammedFailure().thenCompose(
                         (res) -> {
-                            if (!validate()) {
-                                return FutureUtils.exception(new BKException.BKNoSuchLedgerExistsException());
+                            int rc = validate();
+                            if (rc != BKException.Code.OK) {
+                                return FutureUtils.exception(BKException.create(rc));
                             }
 
                             PulsarMockLedgerHandle lh = ledgers.get(ledgerId);

managed-ledger/src/test/java/org/apache/bookkeeper/client/PulsarMockLedgerHandle.java

@@ -29,6 +29,7 @@
 import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.Enumeration;
+import java.util.List;
 import java.util.Queue;
 import java.util.concurrent.CompletableFuture;
 
@@ -38,10 +39,14 @@
 import org.apache.bookkeeper.client.BookKeeper.DigestType;
 import org.apache.bookkeeper.client.api.LastConfirmedAndEntry;
 import org.apache.bookkeeper.client.api.LedgerEntries;
+import org.apache.bookkeeper.client.api.LedgerMetadata;
 import org.apache.bookkeeper.client.api.ReadHandle;
 import org.apache.bookkeeper.client.api.WriteFlag;
 import org.apache.bookkeeper.client.impl.LedgerEntryImpl;
 import org.apache.bookkeeper.common.concurrent.FutureUtils;
+import org.apache.bookkeeper.net.BookieSocketAddress;
+import org.apache.bookkeeper.versioning.LongVersion;
+import org.apache.bookkeeper.versioning.Versioned;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -61,8 +66,8 @@ public class PulsarMockLedgerHandle extends LedgerHandle {
 
     public PulsarMockLedgerHandle(PulsarMockBookKeeper bk, long id,
                            DigestType digest, byte[] passwd) throws GeneralSecurityException {
-        super(bk, id, new LedgerMetadata(3, 3, 2, DigestType.MAC, "".getBytes()), DigestType.MAC, "".getBytes(),
-                EnumSet.noneOf(WriteFlag.class));
+        super(bk.getClientCtx(), id, new Versioned<>(createMetadata(digest, passwd), new LongVersion(0L)),
+              digest, passwd, WriteFlag.NONE);
         this.bk = bk;
         this.id = id;
         this.digest = digest;
@@ -75,6 +80,14 @@ public PulsarMockLedgerHandle(PulsarMockBookKeeper bk, long id,
     public void asyncClose(CloseCallback cb, Object ctx) {
         bk.getProgrammedFailure().thenComposeAsync((res) -> {
                 fenced = true;
+
+                Versioned<LedgerMetadata> current = getVersionedLedgerMetadata();
+                Versioned<LedgerMetadata> newMetadata = new Versioned<>(
+                        LedgerMetadataBuilder.from(current.getValue())
+                        .withClosedState().withLastEntryId(getLastAddConfirmed())
+                        .withLength(getLength()).build(),
+                        new LongVersion(((LongVersion)current.getVersion()).getLongVersion() + 1));
+                setLedgerMetadata(current, newMetadata);
                 return FutureUtils.value(null);
             }, bk.executor).whenCompleteAsync((res, exception) -> {
                     if (exception != null) {
@@ -242,6 +255,18 @@ public CompletableFuture<LastConfirmedAndEntry> readLastAddConfirmedAndEntryAsyn
         return readHandle.readLastAddConfirmedAndEntryAsync(entryId, timeOutInMillis, parallel);
     }
 
+    private static LedgerMetadata createMetadata(DigestType digest, byte[] passwd) {
+        List<BookieSocketAddress> ensemble = Lists.newArrayList(
+                new BookieSocketAddress("192.0.2.1", 1234),
+                new BookieSocketAddress("192.0.2.2", 1234),
+                new BookieSocketAddress("192.0.2.3", 1234));
+        return LedgerMetadataBuilder.create()
+            .withDigestType(digest.toApiDigestType())
+            .withPassword(passwd)
+            .newEnsembleEntry(0L, ensemble)
+            .build();
+    }
+
     private static final Logger log = LoggerFactory.getLogger(PulsarMockLedgerHandle.class);
 
 }

managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java

@@ -223,7 +223,7 @@ public void verifyConcurrentUsage() throws Exception {
     public void testSimple() throws Exception {
         ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc);
         ManagedLedgerConfig mlConfig = new ManagedLedgerConfig();
-        mlConfig.setEnsembleSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1);
+        mlConfig.setEnsembleSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1).setWriteQuorumSize(1);
         // set the data ledger size
         mlConfig.setMaxEntriesPerLedger(100);
         // set the metadata ledger size to 1 to kick off many ledger switching cases
@@ -238,7 +238,9 @@ public void testSimple() throws Exception {
     public void testConcurrentMarkDelete() throws Exception {
         ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc);
         ManagedLedgerConfig mlConfig = new ManagedLedgerConfig();
-        mlConfig.setEnsembleSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataAckQuorumSize(1);
+        mlConfig.setEnsembleSize(1).setWriteQuorumSize(1)
+            .setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataWriteQuorumSize(1)
+            .setMetadataAckQuorumSize(1);
         // set the data ledger size
         mlConfig.setMaxEntriesPerLedger(100);
         // set the metadata ledger size to 1 to kick off many ledger switching cases

managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java

@@ -62,7 +62,6 @@
 import org.apache.bookkeeper.client.LedgerHandle;
 import org.apache.bookkeeper.client.api.LedgerEntries;
 import org.apache.bookkeeper.client.api.ReadHandle;
-import org.apache.bookkeeper.client.LedgerMetadata;
 import org.apache.bookkeeper.client.PulsarMockBookKeeper;
 import org.apache.bookkeeper.client.PulsarMockLedgerHandle;
 import org.apache.bookkeeper.client.api.LedgerEntries;

managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java

@@ -31,6 +31,7 @@
 
 import io.netty.buffer.ByteBuf;
 
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -245,6 +246,9 @@ static class MockMetadata implements LedgerMetadata {
         private final DigestType digestType;
         private final long ctime;
         private final boolean isClosed;
+        private final int metadataFormatVersion;
+        private final State state;
+        private final byte[] password;
         private final Map<String, byte[]> customMetadata;
 
         MockMetadata(LedgerMetadata toCopy) {
@@ -256,10 +260,21 @@ static class MockMetadata implements LedgerMetadata {
             digestType = toCopy.getDigestType();
             ctime = toCopy.getCtime();
             isClosed = toCopy.isClosed();
-
+            metadataFormatVersion = toCopy.getMetadataFormatVersion();
+            state = toCopy.getState();
+            password = Arrays.copyOf(toCopy.getPassword(), toCopy.getPassword().length);
             customMetadata = ImmutableMap.copyOf(toCopy.getCustomMetadata());
         }
 
+        @Override
+        public boolean hasPassword() { return true; }
+
+        @Override
+        public State getState() { return state; }
+
+        @Override
+        public int getMetadataFormatVersion() { return metadataFormatVersion; }
+
         @Override
         public int getEnsembleSize() { return ensembleSize; }
 
@@ -278,6 +293,9 @@ static class MockMetadata implements LedgerMetadata {
         @Override
         public DigestType getDigestType() { return digestType; }
 
+        @Override
+        public byte[] getPassword() { return password; }
+
         @Override
         public long getCtime() { return ctime; }
 
@@ -296,5 +314,10 @@ public List<BookieSocketAddress> getEnsembleAt(long entryId) {
         public NavigableMap<Long, ? extends List<BookieSocketAddress>> getAllEnsembles() {
             throw new UnsupportedOperationException("Pulsar shouldn't look at this");
         }
+
+        @Override
+        public String toSafeString() {
+            return toString();
+        }
     }
 }

managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java

@@ -23,6 +23,9 @@
 
 package org.apache.bookkeeper.test;
 
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.UnpooledByteBufAllocator;
+
 import java.io.File;
 import java.io.IOException;
 import java.net.InetAddress;
@@ -38,7 +41,9 @@
 import java.util.concurrent.Executors;
 import org.apache.bookkeeper.bookie.Bookie;
 import org.apache.bookkeeper.bookie.BookieException;
+import org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage;
 import org.apache.bookkeeper.client.BookKeeperTestClient;
+import org.apache.bookkeeper.common.allocator.PoolingPolicy;
 import org.apache.bookkeeper.conf.AbstractConfiguration;
 import org.apache.bookkeeper.conf.ClientConfiguration;
 import org.apache.bookkeeper.conf.ServerConfiguration;
@@ -202,12 +207,15 @@ protected ServerConfiguration newServerConfiguration(int port, String zkServers,
         conf.setAllowLoopback(true);
         conf.setFlushInterval(60 * 1000);
         conf.setGcWaitTime(60 * 1000);
+        conf.setAllocatorPoolingPolicy(PoolingPolicy.UnpooledHeap);
         String[] ledgerDirNames = new String[ledgerDirs.length];
         for (int i = 0; i < ledgerDirs.length; i++) {
             ledgerDirNames[i] = ledgerDirs[i].getPath();
         }
         conf.setLedgerDirNames(ledgerDirNames);
         conf.setLedgerStorageClass("org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage");
+        conf.setProperty(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB, 4);
+        conf.setProperty(DbLedgerStorage.READ_AHEAD_CACHE_MAX_SIZE_MB, 4);
         return conf;
     }
 
@@ -434,7 +442,7 @@ protected BookieServer startBookie(ServerConfiguration conf) throws Exception {
     protected BookieServer startBookie(ServerConfiguration conf, final Bookie b) throws Exception {
         BookieServer server = new BookieServer(conf) {
             @Override
-            protected Bookie newBookie(ServerConfiguration conf)
+            protected Bookie newBookie(ServerConfiguration conf, ByteBufAllocator allocator)
                     throws IOException, KeeperException, InterruptedException, BookieException {
                 return b;
             }