Merge pull request #6 from apache/master

Merging latest master

build.xml

@@ -28,6 +28,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
     <!-- ====================================================== -->
     <property name="slf4j.version" value="1.7.25"/>
     <property name="commons-cli.version" value="1.2"/>
+    <property name="spotbugsannotations.version" value="3.1.8"/>
 
     <property name="wagon-http.version" value="2.4"/>
     <property name="maven-ant-tasks.version" value="2.1.3"/>
@@ -1705,6 +1706,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant">
                           reportoutputdirectory="${owasp.out.dir}"
                           reportformat="ALL"
                           failBuildOnCVSS="0">
+            <suppressionfile path="${basedir}/owaspSuppressions.xml" />
 
             <fileset dir="${ivy.lib}">
                 <include name="**/*.jar"/>

excludeFindBugsFilter.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<FindBugsFilter>
+    <!-- this work work on JDK11  https://github.com/spotbugs/spotbugs-maven-plugin/issues/92 -->
+    <Bug pattern="RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"/>
+
+    <!-- this problem is to be addressed in ZOOKEEPER-3227 -->
+    <Bug pattern="DM_DEFAULT_ENCODING"/>
+
+    <!-- not really a problem -->
+    <Bug pattern="DM_EXIT"/>
+
+</FindBugsFilter>
+

ivy.xml

@@ -46,6 +46,7 @@
     <dependency org="org.slf4j" name="slf4j-api" rev="${slf4j.version}"/>
     <dependency org="org.slf4j" name="slf4j-log4j12" rev="${slf4j.version}" transitive="false"/>
     <dependency org="commons-cli" name="commons-cli" rev="${commons-cli.version}" />
+    <dependency org="com.github.spotbugs" name="spotbugs-annotations" rev="${spotbugsannotations.version}" />
 
     <dependency org="org.apache.maven.wagon" name="wagon-http" rev="${wagon-http.version}"
                 conf="mvn-ant-task->default"/>
@@ -90,8 +91,8 @@
                 rev="${apache-rat-tasks.version}" conf="releaseaudit->default">
         <exclude org="commons-collections" module="commons-collections"/>
     </dependency>
-    <dependency org="commons-lang" name="commons-lang" 
-                rev="${commons-lang.version}" conf="releaseaudit->default"/>
+    <dependency org="commons-lang" name="commons-lang"
+                rev="${commons-lang.version}"/>
     <dependency org="commons-collections" name="commons-collections" 
                 rev="${commons-collections.version}" conf="releaseaudit->default"/>
     <dependency org="org.owasp" name="dependency-check-ant"

owaspSuppressions.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<suppressions xmlns="https://jeremylong.github.io/DependencyCheck/dependency-suppression.1.2.xsd">
+   <suppress>
+      <!-- ZOOKEEPER-3217 -->
+      <cve>CVE-2018-8088</cve>
+   </suppress>
+</suppressions>

pom.xml

@@ -62,7 +62,6 @@
     <module>zookeeper-server</module>
     <module>zookeeper-client</module>
     <module>zookeeper-recipes</module>
-    <module>zookeeper-contrib</module>
   </modules>
 
   <scm>
@@ -243,6 +242,9 @@
   <profiles>
     <profile>
       <id>full-build</id>
+      <modules>
+        <module>zookeeper-contrib</module>
+      </modules>
     </profile>
     <profile>
       <id>java-build</id>
@@ -276,6 +278,7 @@
     <kerby.version>1.1.0</kerby.version>
     <bouncycastle.version>1.56</bouncycastle.version>
     <commons-collections.version>3.2.2</commons-collections.version>
+    <spotbugsannotations.version>3.1.8</spotbugsannotations.version>
   </properties>
 
   <dependencyManagement>
@@ -408,6 +411,13 @@
         <artifactId>jline</artifactId>
         <version>${jline.version}</version>
       </dependency>
+      <dependency>
+         <groupId>com.github.spotbugs</groupId>
+         <artifactId>spotbugs-annotations</artifactId>
+         <version>${spotbugsannotations.version}</version>
+         <scope>provided</scope>
+         <optional>true</optional>
+        </dependency>
     </dependencies>
   </dependencyManagement>
 
@@ -459,6 +469,14 @@
           <artifactId>clover-maven-plugin</artifactId>
           <version>4.3.1</version>
         </plugin>
+        <plugin>
+          <groupId>com.github.spotbugs</groupId>
+          <artifactId>spotbugs-maven-plugin</artifactId>
+          <version>3.1.8</version>
+          <configuration>
+            <excludeFilterFile>excludeFindBugsFilter.xml</excludeFilterFile>
+          </configuration>
+        </plugin>
       </plugins>
     </pluginManagement>
 
@@ -486,7 +504,11 @@
           </execution>
         </executions>
       </plugin>
-    </plugins>
+      <plugin>
+        <groupId>com.github.spotbugs</groupId>
+        <artifactId>spotbugs-maven-plugin</artifactId>
+      </plugin>
+      </plugins>
   </build>
 
   <reporting>

zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md

@@ -685,6 +685,17 @@ property, when available, is noted below.
     defaults to 1000. This value can only be set as a
     system property.
 
+* *maxResponseCacheSize* :
+    (Java system property: **zookeeper.maxResponseCacheSize**)
+    When set to a positive integer, it determines the size
+    of the cache that stores the serialized form of recently
+    read records. Helps save the serialization cost on
+    popular znodes. The metrics **response_packet_cache_hits**
+    and **response_packet_cache_misses** can be used to tune
+    this value to a given workload. The feature is turned on
+    by default with a value of 400, set to 0 or a negative
+    integer to turn the feature off.
+
 * *autopurge.snapRetainCount* :
     (No Java system property)
     **New in 3.4.0:**
@@ -1270,7 +1281,7 @@ Moving forward, Four Letter Words will be deprecated, please use
 
     $ echo mntr | nc localhost 2185
                   zk_version  3.4.0
-                  zk_avg_latency  0
+                  zk_avg_latency  0.7561              - be account to four decimal places
                   zk_max_latency  0
                   zk_min_latency  0
                   zk_packets_received 70

zookeeper-docs/src/main/resources/markdown/zookeeperInternals.md

@@ -185,7 +185,7 @@ to the leader, the leader will tell the follower to discard U.
 
 A new leader establishes a zxid to start using for new proposals by getting the
 epoch, e, of the highest zxid it has seen and setting the next zxid to use to be
-(e+1, 0), fter the leader syncs with a follower, it will propose a NEW_LEADER
+(e+1, 0), after the leader syncs with a follower, it will propose a NEW_LEADER
 proposal. Once the NEW_LEADER proposal has been committed, the leader will activate
 and start receiving and issuing proposals.
 

zookeeper-docs/src/main/resources/markdown/zookeeperProgrammers.md

@@ -194,7 +194,19 @@ store pointers to the storage locations in ZooKeeper.
 ZooKeeper also has the notion of ephemeral nodes. These znodes
 exists as long as the session that created the znode is active. When
 the session ends the znode is deleted. Because of this behavior
-ephemeral znodes are not allowed to have children.
+ephemeral znodes are not allowed to have children. The list of ephemerals
+for the session can be retrieved using **getEphemerals()** api.
+
+##### getEphemerals()
+Retrieves the list of ephemeral nodes created by the session for the
+given path. If the path is empty, it will list all the ephemeral nodes
+for the session.
+**Use Case** - A sample use case might be, if the list of ephemeral
+nodes for the session need to be collected for duplicate data entry check
+and the nodes are created in sequential manner so you do not know the name
+for duplicate check. In that case, getEphemerals() api could be used to
+get the list of nodes for the session. This might be a typical use case
+for service discovery.
 
 <a name="Sequence+Nodes+--+Unique+Naming"></a>
 

zookeeper-jute/pom.xml

@@ -145,6 +145,14 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <!-- spotbugs does not make sense for generated code -->
+        <groupId>com.github.spotbugs</groupId>
+        <artifactId>spotbugs-maven-plugin</artifactId>
+        <configuration>
+            <skip>true</skip>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 

zookeeper-jute/src/main/java/org/apache/jute/XmlInputArchive.java

@@ -143,6 +143,8 @@ public XmlInputArchive(InputStream in)
         valList = new ArrayList<Value>();
         DefaultHandler handler = new XMLParser(valList);
         SAXParserFactory factory = SAXParserFactory.newInstance();
+        factory.setFeature(javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING, Boolean.TRUE);
+        factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
         SAXParser parser = factory.newSAXParser();
         parser.parse(in, handler);
         vLen = valList.size();

zookeeper-jute/src/main/resources/zookeeper.jute

@@ -222,6 +222,14 @@ module org.apache.zookeeper.proto {
         ustring path;
         int type;
     }
+
+    class GetEphemeralsRequest {
+        ustring prefixPath;
+    }
+
+    class GetEphemeralsResponse {
+        vector<ustring> ephemerals;
+    }
 }
 
 module org.apache.zookeeper.server.quorum {

zookeeper-server/pom.xml

@@ -34,6 +34,12 @@
   <description>ZooKeeper server</description>
 
   <dependencies>
+    <dependency>
+      <groupId>com.github.spotbugs</groupId>
+      <artifactId>spotbugs-annotations</artifactId>
+      <scope>provided</scope>
+      <optional>true</optional>
+    </dependency>
     <dependency>
       <groupId>org.hamcrest</groupId>
       <artifactId>hamcrest-all</artifactId>

zookeeper-server/src/main/java/org/apache/zookeeper/AsyncCallback.java

@@ -328,4 +328,18 @@ interface MultiCallback extends AsyncCallback {
         public void processResult(int rc, String path, Object ctx,
                 List<OpResult> opResults);
     }
+
+    /**
+     * This callback is used to process the getEphemerals results from
+     * a single getEphemerals call.
+     */
+    interface EphemeralsCallback extends AsyncCallback {
+        /**
+         *
+         * @param rc      The return code or the result of the call.
+         * @param ctx     Whatever context object that we passed to asynchronous calls.
+         * @param paths   The path that we passed to asynchronous calls.
+         */
+        public void processResult(int rc, Object ctx, List<String> paths);
+    }
 }

zookeeper-server/src/main/java/org/apache/zookeeper/ClientCnxn.java

@@ -18,6 +18,7 @@
 
 package org.apache.zookeeper;
 
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -52,6 +53,7 @@
 import org.apache.zookeeper.AsyncCallback.ChildrenCallback;
 import org.apache.zookeeper.AsyncCallback.Create2Callback;
 import org.apache.zookeeper.AsyncCallback.DataCallback;
+import org.apache.zookeeper.AsyncCallback.EphemeralsCallback;
 import org.apache.zookeeper.AsyncCallback.MultiCallback;
 import org.apache.zookeeper.AsyncCallback.StatCallback;
 import org.apache.zookeeper.AsyncCallback.StringCallback;
@@ -77,6 +79,7 @@
 import org.apache.zookeeper.proto.GetChildren2Response;
 import org.apache.zookeeper.proto.GetChildrenResponse;
 import org.apache.zookeeper.proto.GetDataResponse;
+import org.apache.zookeeper.proto.GetEphemeralsResponse;
 import org.apache.zookeeper.proto.GetSASLRequest;
 import org.apache.zookeeper.proto.ReplyHeader;
 import org.apache.zookeeper.proto.RequestHeader;
@@ -97,6 +100,7 @@
  * connected to as needed.
  *
  */
+@SuppressFBWarnings({"EI_EXPOSE_REP", "EI_EXPOSE_REP2"})
 public class ClientCnxn {
     private static final Logger LOG = LoggerFactory.getLogger(ClientCnxn.class);
 
@@ -479,6 +483,7 @@ public void queueCallback(AsyncCallback cb, int rc, String path,
             waitingEvents.add(new LocalCallback(cb, rc, path, ctx));
         }
 
+       @SuppressFBWarnings("JLM_JSR166_UTILCONCURRENT_MONITORENTER")
        public void queuePacket(Packet packet) {
           if (wasKilled) {
              synchronized (waitingEvents) {
@@ -495,6 +500,7 @@ public void queueEventOfDeath() {
         }
 
         @Override
+        @SuppressFBWarnings("JLM_JSR166_UTILCONCURRENT_MONITORENTER")
         public void run() {
            try {
               isRunning = true;
@@ -553,6 +559,9 @@ private void processEvent(Object event) {
                     } else if (lcb.cb instanceof StringCallback) {
                         ((StringCallback) lcb.cb).processResult(lcb.rc,
                                 lcb.path, lcb.ctx, null);
+                    } else if (lcb.cb instanceof AsyncCallback.EphemeralsCallback) {
+                        ((AsyncCallback.EphemeralsCallback) lcb.cb).processResult(lcb.rc,
+                              lcb.ctx, null);
                     } else {
                         ((VoidCallback) lcb.cb).processResult(lcb.rc, lcb.path,
                                 lcb.ctx);
@@ -666,7 +675,16 @@ private void processEvent(Object event) {
                 	  } else {
                 		  cb.processResult(rc, clientPath, p.ctx, null);
                 	  }
-                  }  else if (p.cb instanceof VoidCallback) {
+                  } else if (p.response instanceof GetEphemeralsResponse) {
+                    EphemeralsCallback cb = (EphemeralsCallback) p.cb;
+                    GetEphemeralsResponse rsp = (GetEphemeralsResponse) p.response;
+                    if (rc == 0) {
+                      cb.processResult(rc, p.ctx, rsp.getEphemerals());
+                    } else {
+                      cb.processResult(rc, p.ctx, null);
+                    }
+                  }
+                  else if (p.cb instanceof VoidCallback) {
                       VoidCallback cb = (VoidCallback) p.cb;
                       cb.processResult(rc, clientPath, p.ctx);
                   }

zookeeper-server/src/main/java/org/apache/zookeeper/ZKUtil.java

@@ -17,6 +17,7 @@
  */
 package org.apache.zookeeper;
 
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Deque;
@@ -88,6 +89,24 @@ public static void deleteRecursive(ZooKeeper zk, final String pathRoot, VoidCall
             zk.delete(tree.get(i), -1, cb, ctx); //Delete all versions of the node with -1.
         }
     }
+
+    /**
+     * @param filePath the file path to be validated
+     * @return Returns null if valid otherwise error message
+     */
+    public static String validateFileInput(String filePath) {
+        File file = new File(filePath);
+        if (!file.exists()) {
+            return "File '" + file.getAbsolutePath() + "' does not exist.";
+        }
+        if (!file.canRead()) {
+            return "Read permission is denied on the file '" + file.getAbsolutePath() + "'";
+        }
+        if (file.isDirectory()) {
+            return "'" + file.getAbsolutePath() + "' is a direcory. it must be a file.";
+        }
+        return null;
+    }
 
     /**
      * BFS Traversal of the system under pathRoot, with the entries in the list, in the