Issue 1403: ArrayIndexOutOfBoundsException is thrown on readLastAddConfirmedAndEntry

.gitignore

@@ -23,3 +23,6 @@ dependency-reduced-pom.xml
 
 # Logs
 logs/
+
+# Vagrant
+**/.vagrant

bookkeeper-server/src/main/java/org/apache/bookkeeper/client/DistributionSchedule.java

@@ -141,6 +141,13 @@ public WriteSet copy() {
      */
     WriteSet getWriteSet(long entryId);
 
+    /**
+     * Return the set of bookies indices to send the messages to for longpoll reads.
+     *
+     * @param entryId expected next entry id to read.
+     * @return the set of bookies indices to read from.
+     */
+    WriteSet getWriteSetForLongPoll(long entryId);
 
     /**
      * An ack set represents the set of bookies from which

bookkeeper-server/src/main/java/org/apache/bookkeeper/client/ReadLastConfirmedAndEntryOp.java

@@ -81,7 +81,7 @@ abstract class ReadLACAndEntryRequest implements AutoCloseable {
         ReadLACAndEntryRequest(ArrayList<BookieSocketAddress> ensemble, long lId, long eId) {
             this.entryImpl = LedgerEntryImpl.create(lId, eId);
             this.ensemble = ensemble;
-            this.writeSet = lh.distributionSchedule.getWriteSet(eId);
+            this.writeSet = lh.distributionSchedule.getWriteSetForLongPoll(eId);
             if (lh.bk.reorderReadSequence) {
                 this.orderedEnsemble = lh.bk.placementPolicy.reorderReadLACSequence(ensemble,
                         lh.getBookiesHealthInfo(), writeSet.copy());
@@ -435,7 +435,10 @@ boolean complete(int bookieIndex, BookieSocketAddress host, ByteBuf buffer, long
         this.lastAddConfirmed = lh.getLastAddConfirmed();
         this.timeOutInMillis = timeOutInMillis;
         this.numResponsesPending = 0;
-        this.numEmptyResponsesAllowed = getLedgerMetadata().getWriteQuorumSize()
+        // since long poll is effectively reading lac with waits, lac can be potentially
+        // be advanced in different write quorums, so we need to make sure to cover enough
+        // bookies before claiming lac is not advanced.
+        this.numEmptyResponsesAllowed = getLedgerMetadata().getEnsembleSize()
                 - getLedgerMetadata().getAckQuorumSize() + 1;
         this.requestTimeNano = MathUtils.nowInNano();
         this.scheduler = scheduler;

bookkeeper-server/src/main/java/org/apache/bookkeeper/client/RoundRobinDistributionSchedule.java

@@ -53,6 +53,13 @@ public WriteSet getWriteSet(long entryId) {
         return WriteSetImpl.create(ensembleSize, writeQuorumSize, entryId);
     }
 
+    @Override
+    public WriteSet getWriteSetForLongPoll(long entryId) {
+        // for long poll reads, we are trying all the bookies in the ensemble
+        // so we create a `WriteSet` with `writeQuorumSize == ensembleSize`.
+        return WriteSetImpl.create(ensembleSize, ensembleSize /* writeQuorumSize */, entryId);
+    }
+
     @VisibleForTesting
     static WriteSet writeSetFromValues(Integer... values) {
         WriteSetImpl writeSet = WriteSetImpl.create(0, 0, 0);

dev/README.md

@@ -0,0 +1,53 @@
+## Apache BookKeeper Dev Tools
+
+### Running Integration Tests on macOS
+
+Currently all the integration tests are docker based and written using arquillian framework. 
+Due to the networking issues, the integration tests can only be run on linux environment.
+For people who is using macOS as their development environment, you can use [Vagrant](https://www.vagrantup.com)
+to launch a linux virtual machine and run the integration tests there.
+
+1. [Download and Install](https://www.vagrantup.com/downloads.html) Vagrant.
+
+2. Provision and launch the dev vm.
+
+   ```shell
+   $ cd ${BOOKKEEPER_HOME}/dev
+
+   # provision the vm
+   $ vagrant up
+   ```
+
+3. The dev vm will try to mount your current bookkeeper workspace to be under `/bookkeeper` in the vm. You might
+   potentially hit following errors due to fail to install VirtualBox Guest additions.
+
+   ```
+   /sbin/mount.vboxsf: mounting failed with the error: No such device
+   ```
+
+   If that happens, follow the below instructions:
+
+   ```
+   # ssh to the dev vm
+   $ vagrant ssh
+
+   [vagrant@bogon bookkeeper]$ sudo yum update -y
+   [vagrant@bogon bookkeeper]$ exit  
+
+   # reload the vm
+   $ vagrant reload
+   ```
+
+4. Now, you will have a bookkeeper dev vm ready for running integration tests.
+
+   ```shell
+   $ vagrant ssh
+
+   # once you are in the bookkeeper dev vm, you can launch docker.
+   [vagrant@bogon bookkeeper]$ sudo systemctl start docker
+
+   # your bookkeeper workspace will be mount under /bookkeeper
+   [vagrant@bogon bookkeeper]$ cd /bookkeeper
+
+   # you can build and test using maven commands
+   ```

dev/Vagrantfile

@@ -0,0 +1,87 @@
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# All Vagrant configuration is done below. The "2" in Vagrant.configure
+# configures the configuration version (we support older styles for
+# backwards compatibility). Please don't change it unless you know what
+# you're doing.
+Vagrant.configure("2") do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://vagrantcloud.com/search.
+  config.vm.box = "centos/7"
+
+  # Disable automatic box update checking. If you disable this, then
+  # boxes will only be checked for updates when the user runs
+  # `vagrant box outdated`. This is not recommended.
+  # config.vm.box_check_update = false
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  # NOTE: This will enable public access to the opened port
+  # config.vm.network "forwarded_port", guest: 80, host: 8080
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine and only allow access
+  # via 127.0.0.1 to disable public access
+  # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1"
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  # config.vm.network "private_network", ip: "192.168.33.10"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network "public_network"
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
+  config.vm.synced_folder "..", "/bookkeeper"
+
+  # Provider-specific configuration so you can fine-tune various
+  # backing providers for Vagrant. These expose provider-specific options.
+  # Example for VirtualBox:
+  #
+  config.vm.provider "virtualbox" do |vb|
+    vb.name   = "bookkeeper_dev_vm"
+    vb.memory = "4096"
+  end
+  #
+  # View the documentation for the provider you are using for more
+  # information on available options.
+
+  # Enable provisioning with a shell script. Additional provisioners such as
+  # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
+  # documentation for more information about their specific syntax and use.
+  config.vm.provision "shell", inline: <<-SHELL
+    yum update -y
+    yum install -y kernel-devel
+    yum install -y kernel-headers
+    yum install -y wget vim java-1.8.0-openjdk-devel maven docker gcc-c++
+  SHELL
+end