diff --git a/.github/dist-management.diff b/.github/dist-management.diff new file mode 100644 index 00000000000..acb23377359 --- /dev/null +++ b/.github/dist-management.diff @@ -0,0 +1,18 @@ +diff --git a/pom.xml b/pom.xml +index b8c0d9a9d..7c2674e63 100755 +--- a/pom.xml ++++ b/pom.xml +@@ -920,5 +920,12 @@ + + + +- ++ ++ ++ ++ bintray-linkedin-maven ++ linkedin-maven ++ https://api.bintray.com/maven/linkedin/maven/zookeeper/;publish=1;override=1 ++ ++ + diff --git a/.github/workflows/ci-bintray-workflow.yml b/.github/workflows/ci-bintray-workflow.yml new file mode 100644 index 00000000000..9c1ba5a4dbf --- /dev/null +++ b/.github/workflows/ci-bintray-workflow.yml @@ -0,0 +1,49 @@ +# +## Licensed to the Apache Software Foundation (ASF) under one +## or more contributor license agreements. See the NOTICE file +## distributed with this work for additional information +## regarding copyright ownership. The ASF licenses this file +## to you under the Apache License, Version 2.0 (the +## "License"); you may not use this file except in compliance +## with the License. You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, +## software distributed under the License is distributed on an +## "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +## KIND, either express or implied. See the License for the +## specific language governing permissions and limitations +## under the License. +## +name: Publish to Bintray + +on: + release: + branches: + - 'li-dev/**' + types: [published] +jobs: + deploy: + name: deploy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Extract branch name + shell: bash + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF##*/})" + id: extract_branch + - name: Update the version + run: | + sh .github/workflows/version-upgrade.sh ${{ steps.extract_branch.outputs.branch }} + - name: Build with Maven + run: | + mvn clean deploy -DskipTests -s .github/workflows/settings.xml -DretryFailedDeploymentCount=3 + env: + MVN_DEPLOY_BINTRAY_KEY: ${{ secrets.MVN_DEPLOY_BINTRAY_KEY }} + MVN_DEPLOY_BINTRAY_USER: ${{ secrets.MVN_DEPLOY_BINTRAY_USER }} + diff --git a/.github/workflows/settings.xml b/.github/workflows/settings.xml new file mode 100644 index 00000000000..6b72d0c477b --- /dev/null +++ b/.github/workflows/settings.xml @@ -0,0 +1,79 @@ + + + + + + + + + + + + false + + + + bintray-linkedin-maven + + + bintray + + + https://linkedin.bintray.com/maven + + + + + + + + false + + + + bintray-linkedin-maven + + + bintray-plugins + + + https://linkedin.bintray.com/maven + + + + + bintray + + + + + + bintray + + + + + + + bintray-linkedin-maven + ${env.MVN_DEPLOY_BINTRAY_USER} + ${env.MVN_DEPLOY_BINTRAY_KEY} + + + diff --git a/.github/workflows/version-upgrade.sh b/.github/workflows/version-upgrade.sh new file mode 100644 index 00000000000..5a08d67559a --- /dev/null +++ b/.github/workflows/version-upgrade.sh @@ -0,0 +1,52 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Enable below when using locally/manually +# cd ../../ + +echo There are $# arguments to $0: $* +if [ "$#" -eq 2 ]; then + version=$1 + new_version=$2 +else + version=`grep -A 4 "com.linkedin.zookeeper" pom.xml | grep "" | awk 'BEGIN {FS="[<,>]"};{print $3}'` + +# just use the given version as the new version + new_version=$1 + +# Below version upgrade logic is left here just in case + +# minor_version=`echo $version | cut -d'.' -f3` +# major_version=`echo $version | cut -d'.' -f1` # should be 0 +# submajor_version=`echo $version | cut -d'.' -f2` + +# new_minor_version=`expr $minor_version + 1` +# new_version=`echo $version | sed -e "s/${minor_version}/${new_minor_version}/g"` +# new_version="$major_version.$submajor_version.$new_minor_version" +fi +echo "bump up: $version -> $new_version" + +for MODULE in $(find . -name 'pom.xml') +do + echo "bump up $MODULE" + sed -i "s/${version}/${new_version}/g" $MODULE + grep -C 1 "$new_version" $MODULE +done diff --git a/.gitignore b/.gitignore index cf5ff24f045..55937cec57e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ zookeeper-server/version-2/* # SVN .svn .revision -git.properties # Eclipse .metadata diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000000..ab7baed630c --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +pipeline { + agent { + label 'Hadoop' + } + + options { + disableConcurrentBuilds() + buildDiscarder(logRotator(daysToKeepStr: '14')) + timeout(time: 2, unit: 'HOURS') + timestamps() + } + + triggers { + cron('@daily') + } + + stages { + stage('Prepare') { + matrix { + agent any + axes { + axis { + name 'JAVA_VERSION' + values 'JDK 1.8 (latest)', 'JDK 11 (latest)' + } + } + + tools { + maven "Maven (latest)" + jdk "${JAVA_VERSION}" + } + + stages { + stage('BuildAndTest') { + steps { + git 'https://github.com/apache/zookeeper' + sh "git clean -fxd" + sh "mvn verify spotbugs:check checkstyle:check -Pfull-build -Dsurefire-forkcount=4" + } + post { + always { + junit '**/target/surefire-reports/TEST-*.xml' + archiveArtifacts '**/target/*.jar' + } + // Jenkins pipeline jobs fill slaves on PRs without this :( + cleanup() { + script { + sh label: 'Cleanup workspace', script: ''' + # See HADOOP-13951 + chmod -R u+rxw "${WORKSPACE}" + ''' + deleteDir() + } + } + } + } + } + } + } + } +} diff --git a/Jenkinsfile-PreCommit b/Jenkinsfile-PreCommit new file mode 100644 index 00000000000..90b3f99ae17 --- /dev/null +++ b/Jenkinsfile-PreCommit @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +pipeline { + agent { + label 'Hadoop' + } + + options { + disableConcurrentBuilds() + buildDiscarder(logRotator(daysToKeepStr: '14')) + timeout(time: 2, unit: 'HOURS') + timestamps() + } + + tools { + maven "Maven (latest)" + jdk "JDK 1.8 (latest)" + } + + stages { + stage('BuildAndTest') { + steps { + git 'https://github.com/apache/zookeeper' + sh "git clean -fxd" + sh "mvn verify spotbugs:check checkstyle:check -Pfull-build -Dsurefire-forkcount=4" + } + post { + always { + junit '**/target/surefire-reports/TEST-*.xml' + } + } + } + } + + post { + // Jenkins pipeline jobs fill slaves on PRs without this :( + cleanup() { + script { + sh label: 'Cleanup workspace', script: ''' + # See HADOOP-13951 + chmod -R u+rxw "${WORKSPACE}" + ''' + deleteDir() + } + } + } +} diff --git a/NOTICE.txt b/NOTICE.txt index 9ce75ba6097..4c4f8b220fe 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,5 +1,5 @@ Apache ZooKeeper -Copyright 2009-2014 The Apache Software Foundation +Copyright 2009-2020 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/README.md b/README.md index 9a248e12ece..0f9c4b2af13 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,11 @@ is voted on and approved by the Apache ZooKeeper PMC: https://repo1.maven.org/maven2/org/apache/zookeeper/zookeeper/ +## Java 8 + +If you are going to compile with Java 1.8, you should use a +recent release at u211 or above. + # Contributing We always welcome new contributors to the project! See [How to Contribute](https://cwiki.apache.org/confluence/display/ZOOKEEPER/HowToContribute) for details on how to submit patch through pull request and our contribution workflow. diff --git a/README_packaging.md b/README_packaging.md index b290dd9f93d..9c87e4cd483 100644 --- a/README_packaging.md +++ b/README_packaging.md @@ -19,11 +19,11 @@ yum install python-setuptools yum install openssl openssl-devel ``` -On Ubuntu: +On Ubuntu (in case of 16.4+): ``` -apt-get install cppunit -apt-get install python-setuptools +apt-get install libcppunit-dev +apt-get install python-setuptools python2.7-dev apt-get install openssl libssl-dev ``` diff --git a/bin/zkServer.sh b/bin/zkServer.sh index ec3db140fc0..4780118ab32 100755 --- a/bin/zkServer.sh +++ b/bin/zkServer.sh @@ -229,51 +229,75 @@ restart) ;; status) # -q is necessary on some versions of linux where nc returns too quickly, and no stat result is output + isSSL="false" clientPortAddress=`$GREP "^[[:space:]]*clientPortAddress[^[:alpha:]]" "$ZOOCFG" | sed -e 's/.*=//'` if ! [ $clientPortAddress ] then - clientPortAddress="localhost" + clientPortAddress="localhost" fi clientPort=`$GREP "^[[:space:]]*clientPort[^[:alpha:]]" "$ZOOCFG" | sed -e 's/.*=//'` if ! [[ "$clientPort" =~ ^[0-9]+$ ]] then - dataDir=`$GREP "^[[:space:]]*dataDir" "$ZOOCFG" | sed -e 's/.*=//'` - myid=`cat "$dataDir/myid"` - if ! [[ "$myid" =~ ^[0-9]+$ ]] ; then - echo "clientPort not found and myid could not be determined. Terminating." - exit 1 - fi - clientPortAndAddress=`$GREP "^[[:space:]]*server.$myid=.*;.*" "$ZOOCFG" | sed -e 's/.*=//' | sed -e 's/.*;//'` - if [ ! "$clientPortAndAddress" ] ; then - echo "Client port not found in static config file. Looking in dynamic config file." - dynamicConfigFile=`$GREP "^[[:space:]]*dynamicConfigFile" "$ZOOCFG" | sed -e 's/.*=//'` - clientPortAndAddress=`$GREP "^[[:space:]]*server.$myid=.*;.*" "$dynamicConfigFile" | sed -e 's/.*=//' | sed -e 's/.*;//'` - fi - if [ ! "$clientPortAndAddress" ] ; then - echo "Client port not found. Terminating." - exit 1 - fi - if [[ "$clientPortAndAddress" =~ ^.*:[0-9]+ ]] ; then - clientPortAddress=`echo "$clientPortAndAddress" | sed -e 's/:.*//'` - fi - clientPort=`echo "$clientPortAndAddress" | sed -e 's/.*://'` - if [ ! "$clientPort" ] ; then - echo "Client port not found. Terminating." - exit 1 - fi + dataDir=`$GREP "^[[:space:]]*dataDir" "$ZOOCFG" | sed -e 's/.*=//'` + myid=`cat "$dataDir/myid" 2> /dev/null` + if ! [[ "$myid" =~ ^[0-9]+$ ]] ; then + echo "myid could not be determined, will not able to locate clientPort in the server configs." + else + clientPortAndAddress=`$GREP "^[[:space:]]*server.$myid=.*;.*" "$ZOOCFG" | sed -e 's/.*=//' | sed -e 's/.*;//'` + if [ ! "$clientPortAndAddress" ] ; then + echo "Client port not found in static config file. Looking in dynamic config file." + dynamicConfigFile=`$GREP "^[[:space:]]*dynamicConfigFile" "$ZOOCFG" | sed -e 's/.*=//'` + clientPortAndAddress=`$GREP "^[[:space:]]*server.$myid=.*;.*" "$dynamicConfigFile" | sed -e 's/.*=//' | sed -e 's/.*;//'` + fi + if [ ! "$clientPortAndAddress" ] ; then + echo "Client port not found in the server configs" + else + if [[ "$clientPortAndAddress" =~ ^.*:[0-9]+ ]] ; then + if [[ "$clientPortAndAddress" =~ \[.*\]:[0-9]+ ]] ; then + # Extracts address from address:port for example extracts 127::1 from "[127::1]:2181" + clientPortAddress=`echo "$clientPortAndAddress" | sed -e 's|\[||' | sed -e 's|\]:.*||'` + else + clientPortAddress=`echo "$clientPortAndAddress" | sed -e 's/:.*//'` + fi + fi + clientPort=`echo "$clientPortAndAddress" | sed -e 's/.*://'` + fi + fi fi - echo "Client port found: $clientPort. Client address: $clientPortAddress." + if [ ! "$clientPort" ] ; then + echo "Client port not found. Looking for secureClientPort in the static config." + secureClientPort=`$GREP "^[[:space:]]*secureClientPort[^[:alpha:]]" "$ZOOCFG" | sed -e 's/.*=//'` + if [ "$secureClientPort" ] ; then + isSSL="true" + clientPort=$secureClientPort + else + echo "Unable to find either secure or unsecure client port in any configs. Terminating." + exit 1 + fi + fi + echo "Client port found: $clientPort. Client address: $clientPortAddress. Client SSL: $isSSL." STAT=`"$JAVA" "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" "-Dzookeeper.log.file=${ZOO_LOG_FILE}" \ - -cp "$CLASSPATH" $JVMFLAGS org.apache.zookeeper.client.FourLetterWordMain \ - $clientPortAddress $clientPort srvr 2> /dev/null \ + -cp "$CLASSPATH" $CLIENT_JVMFLAGS $JVMFLAGS org.apache.zookeeper.client.FourLetterWordMain \ + $clientPortAddress $clientPort srvr $isSSL 2> /dev/null \ | $GREP Mode` if [ "x$STAT" = "x" ] then - echo "Error contacting service. It is probably not running." - exit 1 + if [ "$isSSL" = "true" ] ; then + echo " " + echo "Note: We used secureClientPort ($secureClientPort) to establish connection, but we failed. The 'status'" + echo " command establishes a client connection to the server to execute diagnostic commands. Please make sure you" + echo " provided all the Client SSL connection related parameters in the CLIENT_JVMFLAGS environment variable! E.g.:" + echo " CLIENT_JVMFLAGS=\"-Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty" + echo " -Dzookeeper.ssl.trustStore.location=/tmp/clienttrust.jks -Dzookeeper.ssl.trustStore.password=password" + echo " -Dzookeeper.ssl.keyStore.location=/tmp/client.jks -Dzookeeper.ssl.keyStore.password=password" + echo " -Dzookeeper.client.secure=true\" ./zkServer.sh status" + echo " " + fi + echo "Error contacting service. It is probably not running." + exit 1 else - echo $STAT - exit 0 + echo $STAT + exit 0 fi ;; *) diff --git a/checkstyleSuppressions.xml b/checkstyleSuppressions.xml index 53d92eb5bda..0a270041419 100644 --- a/checkstyleSuppressions.xml +++ b/checkstyleSuppressions.xml @@ -32,4 +32,6 @@ + + diff --git a/owaspSuppressions.xml b/owaspSuppressions.xml index 5c4bc33bcf5..2565f0db148 100644 --- a/owaspSuppressions.xml +++ b/owaspSuppressions.xml @@ -41,4 +41,14 @@ this writing --> CVE-2019-3826 + + + CVE-2019-17571 + + + + CVE-2020-9488 + diff --git a/pom.xml b/pom.xml index c501bcdac7f..6ba5a5963de 100755 --- a/pom.xml +++ b/pom.xml @@ -23,17 +23,19 @@ org.apache apache - 21 + 23 - org.apache.zookeeper + com.linkedin.zookeeper parent pom - 3.6.0-SNAPSHOT + 3.6.2 Apache ZooKeeper + Disclaimer: this version of ZooKeeper artifact has been modified for use at LinkedIn. + ZooKeeper is a centralized service for maintaining configuration information, naming, providing distributed synchronization, and providing group services. All of these kinds of services are used in some form or another by distributed applications. Each time they @@ -64,56 +66,19 @@ zookeeper-client zookeeper-recipes zookeeper-assembly + zookeeper-compatibility-tests scm:git:https://gitbox.apache.org/repos/asf/zookeeper.git scm:git:https://gitbox.apache.org/repos/asf/zookeeper.git https://gitbox.apache.org/repos/asf/zookeeper.git - branch-3.6 + release-3.6.2-1 - JIRA - http://issues.apache.org/jira/browse/ZOOKEEPER + GitHub Issues + https://github.com/linkedin/zookeeper/issues - - jenkins - https://builds.apache.org/view/S-Z/view/ZooKeeper/ - - - - User List - user-subscribe@zookeeper.apache.org - user-unsubscribe@zookeeper.apache.org - user@zookeeper.apache.org - http://mail-archives.apache.org/mod_mbox/zookeeper-user/ - - - Developer List - dev-subscribe@zookeeper.apache.org - dev-unsubscribe@zookeeper.apache.org - dev@zookeeper.apache.org - http://mail-archives.apache.org/mod_mbox/zookeeper-dev/ - - - Commits List - commits-subscribe@zookeeper.apache.org - commits-unsubscribe@zookeeper.apache.org - http://mail-archives.apache.org/mod_mbox/zookeeper-commits/ - - - Issues List - issues-subscribe@zookeeper.apache.org - issues-unsubscribe@zookeeper.apache.org - https://lists.apache.org/list.html?issues@zookeeper.apache.org - - - Notifications List - notifications-subscribe@zookeeper.apache.org - notifications-unsubscribe@zookeeper.apache.org - https://lists.apache.org/list.html?notifications@zookeeper.apache.org - - tdunning @@ -258,6 +223,14 @@ full-build + zookeeper-it + zookeeper-contrib + + + + fatjar + + zookeeper-it zookeeper-contrib @@ -267,6 +240,66 @@ true + + apache-release + + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.apache.resources + apache-source-release-assembly-descriptor + 1.0.6 + + + + + source-release-assembly-tar-gz + initialize + + single + + + true + + + ${sourceReleaseAssemblyDescriptor} + + apache-zookeeper-${project.version} + false + + tar.gz + + posix + + + + + + + + + m2e + + + m2e.version + + + + 8 + + + + jdk-release-flag + + [9,) + + + 8 + + @@ -280,23 +313,24 @@ 1.7.25 0.5.0 + 1.48 4.12 1.2.17 2.27.0 1.3 1.2 - 4.1.42.Final - 9.4.18.v20190429 - 2.9.10.1 + 4.1.50.Final + 9.4.24.v20191120 + 2.10.3 1.1.1 - 2.11 + 2.14.6 1.1.7 - 1.1.0 + 2.0.0 1.60 3.2.2 2.6 3.2.5 - 3.1.9 + 4.0.2 8.17 @@ -399,6 +433,11 @@ log4j ${log4j.version} + + org.jmockit + jmockit + ${jmockit.version} + junit junit @@ -479,15 +518,10 @@ - - org.codehaus.mojo - properties-maven-plugin - 1.0.0 - org.apache.maven.plugins maven-compiler-plugin - 3.8.0 + 3.8.1 true @@ -505,6 +539,13 @@ org.apache.maven.plugins maven-jar-plugin 3.1.0 + + + + ${mvngit.commit.id} + + + org.apache.maven.plugins @@ -555,10 +596,12 @@ 1.6.0 - pl.project13.maven - git-commit-id-plugin - 2.2.5 - false + com.github.koraktor + mavanagaiata + 0.9.4 + + true + org.codehaus.mojo @@ -578,7 +621,7 @@ com.github.spotbugs spotbugs-maven-plugin - 3.1.9 + 4.0.0 excludeFindBugsFilter.xml @@ -586,7 +629,7 @@ org.owasp dependency-check-maven - 5.2.4 + 5.3.0 org.apache.maven.plugins @@ -619,38 +662,32 @@ + + + maven-remote-resources-plugin + + + process-resource-bundles + none + + + - pl.project13.maven - git-commit-id-plugin + com.github.koraktor + mavanagaiata find-current-git-revision - revision + commit validate - - false - true - false - ${project.basedir}/.git - git - false - true - ${project.basedir}/zookeeper-server/src/main/resources/git.properties - properties - - false - false - -dirty - - org.openclover @@ -675,7 +712,6 @@ - org.apache.maven.plugins maven-antrun-plugin @@ -792,7 +828,6 @@ **/log4j.properties **/README.md - **/git.properties **/findbugsExcludeFile.xml **/checkstyle-noframes-sorted.xsl **/configure.ac @@ -806,22 +841,22 @@ src/main/resources/markdown/html/* src/main/resources/markdown/images/* - zookeeper-contrib-monitoring/JMX-RESOURCES - zookeeper-contrib-fatjar/src/main/resources/mainClasses - zookeeper-contrib-zkperl/Changes - zookeeper-contrib-zkperl/MANIFEST - zookeeper-contrib-loggraph/src/main/resources/webapp/org/apache/zookeeper/graph/resources/* - src/main/resources/webapp/org/apache/zookeeper/graph/resources/* - src/main/java/com/nitido/utils/toaster/Toaster.java - TODO + **/JMX-RESOURCES + **/src/main/resources/mainClasses + **/Changes + **/MANIFEST + **/src/test/zoo.cfg + **/src/main/resources/webapp/org/apache/zookeeper/graph/resources/* + **/src/main/java/com/nitido/utils/toaster/Toaster.java + **/TODO **/acinclude.m4 **/aminclude.am - src/hashtable/* - include/winconfig.h - tests/wrappers.opt - tests/zoo.cfg - tests/wrappers-mt.opt + **/src/hashtable/* + **/include/winconfig.h + **/tests/wrappers.opt + **/tests/zoo.cfg + **/tests/wrappers-mt.opt **/c-doc.Doxyfile true @@ -887,4 +922,11 @@ + + + bintray-linkedin-maven + linkedin-maven + https://api.bintray.com/maven/linkedin/maven/zookeeper/;publish=1;override=1 + + diff --git a/zk-merge-pr.py b/zk-merge-pr.py index 5768a94d74f..dc5549d9dfa 100644 --- a/zk-merge-pr.py +++ b/zk-merge-pr.py @@ -32,7 +32,7 @@ import re import subprocess import sys -import urllib2 +import urllib.request, urllib.error, urllib.parse import getpass try: @@ -75,48 +75,48 @@ def get_json(url): try: - request = urllib2.Request(url) + request = urllib.request.Request(url) if GITHUB_OAUTH_KEY: request.add_header('Authorization', 'token %s' % GITHUB_OAUTH_KEY) - return json.load(urllib2.urlopen(request)) - except urllib2.HTTPError as e: + return json.load(urllib.request.urlopen(request)) + except urllib.error.HTTPError as e: if "X-RateLimit-Remaining" in e.headers and e.headers["X-RateLimit-Remaining"] == '0': - print "Exceeded the GitHub API rate limit; see the instructions in " + \ + print("Exceeded the GitHub API rate limit; see the instructions in " + \ "zk-merge-pr.py to configure an OAuth token for making authenticated " + \ - "GitHub requests." + "GitHub requests.") else: - print "Unable to fetch URL, exiting: %s" % url + print("Unable to fetch URL, exiting: %s" % url) sys.exit(-1) def fail(msg): - print msg + print(msg) clean_up() sys.exit(-1) def run_cmd(cmd): - print cmd + print(cmd) if isinstance(cmd, list): - return subprocess.check_output(cmd) + return subprocess.check_output(cmd, encoding='utf8') else: - return subprocess.check_output(cmd.split(" ")) + return subprocess.check_output(cmd.split(" "), encoding='utf8') def continue_maybe(prompt): - result = raw_input("\n%s (y/n): " % prompt) + result = input("\n%s (y/n): " % prompt) if result.lower().strip() != "y": fail("Okay, exiting") def clean_up(): if original_head != get_current_branch(): - print "Restoring head pointer to %s" % original_head + print("Restoring head pointer to %s" % original_head) run_cmd("git checkout %s" % original_head) branches = run_cmd("git branch").replace(" ", "").split("\n") - for branch in filter(lambda x: x.startswith(TEMP_BRANCH_PREFIX), branches): - print "Deleting local branch %s" % branch + for branch in [x for x in branches if x.startswith(TEMP_BRANCH_PREFIX)]: + print("Deleting local branch %s" % branch) run_cmd("git branch -D %s" % branch) def get_current_branch(): @@ -144,20 +144,20 @@ def merge_pr(pr_num, target_ref, title, body, pr_repo_desc): '--pretty=format:%an <%ae>']).split("\n") distinct_authors = sorted(set(commit_authors), key=lambda x: commit_authors.count(x), reverse=True) - primary_author = raw_input( + primary_author = input( "Enter primary author in the format of \"name \" [%s]: " % distinct_authors[0]) if primary_author == "": primary_author = distinct_authors[0] - reviewers = raw_input( + reviewers = input( "Enter reviewers in the format of \"name1 , name2 \": ").strip() commits = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name, '--pretty=format:%h [%an] %s']).split("\n") if len(commits) > 1: - result = raw_input("List pull request commits in squashed commit message? (y/n): ") + result = input("List pull request commits in squashed commit message? (y/n): ") if result.lower().strip() == "y": should_list_commits = True else: @@ -209,13 +209,13 @@ def merge_pr(pr_num, target_ref, title, body, pr_repo_desc): merge_hash = run_cmd("git rev-parse %s" % target_branch_name)[:8] clean_up() - print("Pull request #%s merged!" % pr_num) - print("Merge hash: %s" % merge_hash) + print(("Pull request #%s merged!" % pr_num)) + print(("Merge hash: %s" % merge_hash)) return merge_hash def cherry_pick(pr_num, merge_hash, default_branch): - pick_ref = raw_input("Enter a branch name [%s]: " % default_branch) + pick_ref = input("Enter a branch name [%s]: " % default_branch) if pick_ref == "": pick_ref = default_branch @@ -244,21 +244,21 @@ def cherry_pick(pr_num, merge_hash, default_branch): pick_hash = run_cmd("git rev-parse %s" % pick_branch_name)[:8] clean_up() - print("Pull request #%s picked into %s!" % (pr_num, pick_ref)) - print("Pick hash: %s" % pick_hash) + print(("Pull request #%s picked into %s!" % (pr_num, pick_ref))) + print(("Pick hash: %s" % pick_hash)) return pick_ref def fix_version_from_branch(branch, versions): # Note: Assumes this is a sorted (newest->oldest) list of un-released versions if branch == DEV_BRANCH_NAME: - versions = filter(lambda x: x == DEFAULT_FIX_VERSION, versions) + versions = [x for x in versions if x == DEFAULT_FIX_VERSION] if len(versions) > 0: return versions[0] else: return None else: - versions = filter(lambda x: x.startswith(branch), versions) + versions = [x for x in versions if x.startswith(branch)] if len(versions) > 0: return versions[-1] else: @@ -269,7 +269,7 @@ def resolve_jira_issue(merge_branches, comment, default_jira_id=""): asf_jira = jira.client.JIRA({'server': JIRA_API_BASE}, basic_auth=(JIRA_USERNAME, JIRA_PASSWORD)) - jira_id = raw_input("Enter a JIRA id [%s]: " % default_jira_id) + jira_id = input("Enter a JIRA id [%s]: " % default_jira_id) if jira_id == "": jira_id = default_jira_id @@ -288,20 +288,20 @@ def resolve_jira_issue(merge_branches, comment, default_jira_id=""): if cur_status == "Resolved" or cur_status == "Closed": fail("JIRA issue %s already has status '%s'" % (jira_id, cur_status)) - print ("=== JIRA %s ===" % jira_id) - print ("summary\t\t%s\nassignee\t%s\nstatus\t\t%s\nurl\t\t%s/%s\n" % ( - cur_summary, cur_assignee, cur_status, JIRA_BASE, jira_id)) + print(("=== JIRA %s ===" % jira_id)) + print(("summary\t\t%s\nassignee\t%s\nstatus\t\t%s\nurl\t\t%s/%s\n" % ( + cur_summary, cur_assignee, cur_status, JIRA_BASE, jira_id))) versions = asf_jira.project_versions(CAPITALIZED_PROJECT_NAME) versions = sorted(versions, key=lambda x: x.name, reverse=True) - versions = filter(lambda x: x.raw['released'] is False, versions) + versions = [x for x in versions if x.raw['released'] is False] - version_names = map(lambda x: x.name, versions) - default_fix_versions = map(lambda x: fix_version_from_branch(x, version_names), merge_branches) - default_fix_versions = filter(lambda x: x != None, default_fix_versions) + version_names = [x.name for x in versions] + default_fix_versions = [fix_version_from_branch(x, version_names) for x in merge_branches] + default_fix_versions = [x for x in default_fix_versions if x != None] default_fix_versions = ",".join(default_fix_versions) - fix_versions = raw_input("Enter comma-separated fix version(s) [%s]: " % default_fix_versions) + fix_versions = input("Enter comma-separated fix version(s) [%s]: " % default_fix_versions) if fix_versions == "": fix_versions = default_fix_versions fix_versions = fix_versions.replace(" ", "").split(",") @@ -309,7 +309,7 @@ def resolve_jira_issue(merge_branches, comment, default_jira_id=""): def get_version_json(version_str): return filter(lambda v: v.name == version_str, versions)[0].raw - jira_fix_versions = map(lambda v: get_version_json(v), fix_versions) + jira_fix_versions = [get_version_json(v) for v in fix_versions] resolve = filter(lambda a: a['name'] == "Resolve Issue", asf_jira.transitions(jira_id))[0] resolution = filter(lambda r: r.raw['name'] == "Fixed", asf_jira.resolutions())[0] @@ -317,7 +317,7 @@ def get_version_json(version_str): jira_id, resolve["id"], fixVersions = jira_fix_versions, comment = comment, resolution = {'id': resolution.raw['id']}) - print "Successfully resolved %s with fixVersions=%s!" % (jira_id, fix_versions) + print("Successfully resolved %s with fixVersions=%s!" % (jira_id, fix_versions)) def resolve_jira_issues(title, merge_branches, comment): @@ -401,7 +401,7 @@ def check_git_remote(): # check if all remote endpoints' URLs point to project git repo name = PROJECT_NAME + ".git" - for url in repos.values(): + for url in list(repos.values()): if not url.endswith(name): fail("Error: not a %s git repo or at least one remote is invalid" % PROJECT_NAME) @@ -419,7 +419,7 @@ def check_jira_env(): if JIRA_IMPORTED: if JIRA_USERNAME.strip() != "" and JIRA_PASSWORD.strip() == "": - inform_pwd = raw_input("JIRA_USERNAME set but JIRA_PASSWORD is not. Want to inform it? ") + inform_pwd = input("JIRA_USERNAME set but JIRA_PASSWORD is not. Want to inform it? ") if inform_pwd.strip() == "y": JIRA_PASSWORD = getpass.getpass('JIRA PASSWORD: ') @@ -440,34 +440,34 @@ def main(): check_git_remote() branches = get_json("%s/branches" % GITHUB_API_BASE) - branch_names = filter(lambda x: x.startswith(RELEASE_BRANCH_PREFIX), [x['name'] for x in branches]) + branch_names = [x for x in [x['name'] for x in branches] if x.startswith(RELEASE_BRANCH_PREFIX)] # Assumes branch names can be sorted lexicographically latest_branch = sorted(branch_names, reverse=True)[0] - pr_num = raw_input("Which pull request would you like to merge? (e.g. 34): ") + pr_num = input("Which pull request would you like to merge? (e.g. 34): ") pr = get_json("%s/pulls/%s" % (GITHUB_API_BASE, pr_num)) pr_events = get_json("%s/issues/%s/events" % (GITHUB_API_BASE, pr_num)) url = pr["url"] pr_title = pr["title"] - commit_title = raw_input("Commit title [%s]: " % pr_title.encode("utf-8")).decode("utf-8") + commit_title = input("Commit title [%s]: " % pr_title) if commit_title == "": commit_title = pr_title # Decide whether to use the modified title or not modified_title = standardize_jira_ref(commit_title) if modified_title != commit_title: - print "I've re-written the title as follows to match the standard format:" - print "Original: %s" % commit_title - print "Modified: %s" % modified_title - result = raw_input("Would you like to use the modified title? (y/n): ") + print("I've re-written the title as follows to match the standard format:") + print("Original: %s" % commit_title) + print("Modified: %s" % modified_title) + result = input("Would you like to use the modified title? (y/n): ") if result.lower().strip() == "y": commit_title = modified_title - print "Using modified title:" + print("Using modified title:") else: - print "Using original title:" - print commit_title + print("Using original title:") + print(commit_title) body = pr["body"] target_ref = pr["base"]["ref"] @@ -484,13 +484,13 @@ def main(): merge_hash = merge_commits[0]["commit_id"] message = get_json("%s/commits/%s" % (GITHUB_API_BASE, merge_hash))["commit"]["message"] - print "Pull request %s has already been merged, assuming you want to backport" % pr_num + print("Pull request %s has already been merged, assuming you want to backport" % pr_num) commit_is_downloaded = run_cmd(['git', 'rev-parse', '--quiet', '--verify', "%s^{commit}" % merge_hash]).strip() != "" if not commit_is_downloaded: fail("Couldn't find any merge commit for #%s, you may need to update HEAD." % pr_num) - print "Found commit %s:\n%s" % (merge_hash, message) + print("Found commit %s:\n%s" % (merge_hash, message)) cherry_pick(pr_num, merge_hash, latest_branch) sys.exit(0) @@ -499,9 +499,9 @@ def main(): "Continue? (experts only!)" continue_maybe(msg) - print ("\n=== Pull Request #%s ===" % pr_num) - print ("PR title\t%s\nCommit title\t%s\nSource\t\t%s\nTarget\t\t%s\nURL\t\t%s" % ( - pr_title, commit_title, pr_repo_desc, target_ref, url)) + print(("\n=== Pull Request #%s ===" % pr_num)) + print(("PR title\t%s\nCommit title\t%s\nSource\t\t%s\nTarget\t\t%s\nURL\t\t%s" % ( + pr_title, commit_title, pr_repo_desc, target_ref, url))) continue_maybe("Proceed with merging pull request #%s?" % pr_num) merged_refs = [target_ref] @@ -509,7 +509,7 @@ def main(): merge_hash = merge_pr(pr_num, target_ref, commit_title, body, pr_repo_desc) pick_prompt = "Would you like to pick %s into another branch?" % merge_hash - while raw_input("\n%s (y/n): " % pick_prompt).lower().strip() == "y": + while input("\n%s (y/n): " % pick_prompt).lower().strip() == "y": merged_refs = merged_refs + [cherry_pick(pr_num, merge_hash, latest_branch)] if JIRA_IMPORTED: @@ -518,11 +518,11 @@ def main(): jira_comment = "Issue resolved by pull request %s\n[%s/%s]" % (pr_num, GITHUB_BASE, pr_num) resolve_jira_issues(commit_title, merged_refs, jira_comment) else: - print "JIRA_USERNAME and JIRA_PASSWORD not set" - print "Exiting without trying to close the associated JIRA." + print("JIRA_USERNAME and JIRA_PASSWORD not set") + print("Exiting without trying to close the associated JIRA.") else: - print "Could not find jira-python library. Run 'sudo pip install jira' to install." - print "Exiting without trying to close the associated JIRA." + print("Could not find jira-python library. Run 'sudo pip install jira' to install.") + print("Exiting without trying to close the associated JIRA.") if __name__ == "__main__": import doctest diff --git a/zookeeper-assembly/pom.xml b/zookeeper-assembly/pom.xml index 36964956c98..92caeb94b95 100755 --- a/zookeeper-assembly/pom.xml +++ b/zookeeper-assembly/pom.xml @@ -21,13 +21,12 @@ --> 4.0.0 - org.apache.zookeeper + com.linkedin.zookeeper parent - 3.6.0-SNAPSHOT + 3.6.2 .. - org.apache.zookeeper zookeeper-assembly pom Apache ZooKeeper - Assembly @@ -51,33 +50,33 @@ - org.apache.zookeeper + com.linkedin.zookeeper zookeeper-docs ${project.version} - org.apache.zookeeper + com.linkedin.zookeeper zookeeper-jute ${project.version} - org.apache.zookeeper + com.linkedin.zookeeper zookeeper ${project.version} - org.apache.zookeeper + com.linkedin.zookeeper zookeeper-client ${project.version} pom - org.apache.zookeeper + com.linkedin.zookeeper zookeeper-prometheus-metrics ${project.version} - org.apache.zookeeper + com.linkedin.zookeeper zookeeper-recipes ${project.version} pom @@ -124,21 +123,6 @@ org.apache.maven.plugins maven-assembly-plugin - - source-assembly - package - - single - - - - ${project.basedir}/src/main/assembly/source-package.xml - - apache-zookeeper-${project.version} - false - posix - - bin-assembly package @@ -191,4 +175,4 @@ - \ No newline at end of file + diff --git a/zookeeper-assembly/src/main/assembly/bin-package.xml b/zookeeper-assembly/src/main/assembly/bin-package.xml index 1fa81d0bb22..970160d3a86 100644 --- a/zookeeper-assembly/src/main/assembly/bin-package.xml +++ b/zookeeper-assembly/src/main/assembly/bin-package.xml @@ -35,9 +35,9 @@ *:* - org.apache.zookeeper:zookeeper-recipes - org.apache.zookeeper:zookeeper-client - org.apache.zookeeper:zookeeper-docs + com.linkedin.zookeeper:zookeeper-recipes + com.linkedin.zookeeper:zookeeper-client + com.linkedin.zookeeper:zookeeper-docs false true diff --git a/zookeeper-assembly/src/main/assembly/source-package.xml b/zookeeper-assembly/src/main/assembly/source-package.xml deleted file mode 100644 index 15a5c68df79..00000000000 --- a/zookeeper-assembly/src/main/assembly/source-package.xml +++ /dev/null @@ -1,118 +0,0 @@ - - - source-package - - tar.gz - - true - - src/main/assembly/components.xml - - - - - ${project.basedir}/../zookeeper-assembly - - **/target/** - - zookeeper-assembly - ${rw.file.permission} - ${rwx.file.permission} - - - ${project.basedir}/../zookeeper-client - - **/target/** - - zookeeper-client - ${rw.file.permission} - ${rwx.file.permission} - - - ${project.basedir}/../zookeeper-contrib - - **/target/** - - zookeeper-contrib - ${rw.file.permission} - ${rwx.file.permission} - - - ${project.basedir}/../zookeeper-docs - - **/target/** - - zookeeper-docs - ${rw.file.permission} - ${rwx.file.permission} - - - ${project.basedir}/../zookeeper-it - - **/target/** - - zookeeper-it - ${rw.file.permission} - ${rwx.file.permission} - - - ${project.basedir}/../zookeeper-jute - - **/target/** - - zookeeper-jute - ${rw.file.permission} - ${rwx.file.permission} - - - ${project.basedir}/../zookeeper-recipes - - **/target/** - - zookeeper-recipes - ${rw.file.permission} - ${rwx.file.permission} - - - ${project.basedir}/../zookeeper-server - - **/target/** - - zookeeper-server - ${rw.file.permission} - ${rwx.file.permission} - - - ${project.basedir}/.. - . - - pom.xml - excludeFindBugsFilter.xml - owaspSuppressions.xml - checktyle.xml - checktyleSuppressions.xml - - ${rw.file.permission} - - - diff --git a/zookeeper-client/pom.xml b/zookeeper-client/pom.xml index e05ef4c37e3..1dc7cce6669 100755 --- a/zookeeper-client/pom.xml +++ b/zookeeper-client/pom.xml @@ -21,20 +21,17 @@ --> 4.0.0 - org.apache.zookeeper + com.linkedin.zookeeper parent - 3.6.0-SNAPSHOT + 3.6.2 .. - org.apache.zookeeper zookeeper-client pom Apache ZooKeeper - Client ZooKeeper client - - full-build diff --git a/zookeeper-client/zookeeper-client-c/CMakeLists.txt b/zookeeper-client/zookeeper-client-c/CMakeLists.txt index 06bbf983912..1350daa36a2 100644 --- a/zookeeper-client/zookeeper-client-c/CMakeLists.txt +++ b/zookeeper-client/zookeeper-client-c/CMakeLists.txt @@ -16,7 +16,7 @@ cmake_minimum_required(VERSION 3.5) -project(zookeeper VERSION 3.6.0) +project(zookeeper VERSION 3.6.2) set(email user@zookeeper.apache.org) set(description "zookeeper C client") diff --git a/zookeeper-client/zookeeper-client-c/configure.ac b/zookeeper-client/zookeeper-client-c/configure.ac index 96ddaeca03e..0aec9569ff4 100644 --- a/zookeeper-client/zookeeper-client-c/configure.ac +++ b/zookeeper-client/zookeeper-client-c/configure.ac @@ -3,7 +3,7 @@ AC_PREREQ(2.59) -AC_INIT([zookeeper C client],3.6.0,[user@zookeeper.apache.org],[zookeeper]) +AC_INIT([zookeeper C client],3.6.2,[user@zookeeper.apache.org],[zookeeper]) AC_CONFIG_SRCDIR([src/zookeeper.c]) # Save initial CFLAGS and CXXFLAGS values before AC_PROG_CC and AC_PROG_CXX diff --git a/zookeeper-client/zookeeper-client-c/pom.xml b/zookeeper-client/zookeeper-client-c/pom.xml index 3536d1a3c27..3dcd4860d08 100755 --- a/zookeeper-client/zookeeper-client-c/pom.xml +++ b/zookeeper-client/zookeeper-client-c/pom.xml @@ -21,13 +21,12 @@ --> 4.0.0 - org.apache.zookeeper + com.linkedin.zookeeper zookeeper-client - 3.6.0-SNAPSHOT + 3.6.2 .. - org.apache.zookeeper zookeeper-client-c jar Apache ZooKeeper - Client - C diff --git a/zookeeper-client/zookeeper-client-c/src/addrvec.c b/zookeeper-client/zookeeper-client-c/src/addrvec.c index fdfb68d34fd..b7f244e7afb 100644 --- a/zookeeper-client/zookeeper-client-c/src/addrvec.c +++ b/zookeeper-client/zookeeper-client-c/src/addrvec.c @@ -126,8 +126,26 @@ int addrvec_contains(const addrvec_t *avec, const struct sockaddr_storage *addr) for (i = 0; i < avec->count; i++) { - if(memcmp(&avec->data[i], addr, INET_ADDRSTRLEN) == 0) - return 1; + if (avec->data[i].ss_family != addr->ss_family) + continue; + switch (addr->ss_family) { + case AF_INET: + if (memcmp(&((struct sockaddr_in*)&avec->data[i])->sin_addr, + &((struct sockaddr_in*)addr)->sin_addr, + sizeof(struct in_addr)) == 0) + return 1; + break; +#ifdef AF_INET6 + case AF_INET6: + if (memcmp(&((struct sockaddr_in6*)&avec->data[i])->sin6_addr, + &((struct sockaddr_in6*)addr)->sin6_addr, + sizeof(struct in6_addr)) == 0) + return 1; + break; +#endif + default: + break; + } } return 0; diff --git a/zookeeper-client/zookeeper-client-c/src/cli.c b/zookeeper-client/zookeeper-client-c/src/cli.c index b2314345aee..34f2b99ef7e 100644 --- a/zookeeper-client/zookeeper-client-c/src/cli.c +++ b/zookeeper-client/zookeeper-client-c/src/cli.c @@ -736,6 +736,19 @@ int handleBatchMode(const char* arg, const char** buf) { return 1; } +#ifdef THREADED +static void millisleep(int ms) { +#ifdef WIN32 + Sleep(ms); +#else /* !WIN32 */ + struct timespec ts; + ts.tv_sec = ms / 1000; + ts.tv_nsec = (ms % 1000) * 1000000; // to nanoseconds + nanosleep(&ts, NULL); +#endif /* WIN32 */ +} +#endif /* THREADED */ + int main(int argc, char **argv) { static struct option long_options[] = { {"host", required_argument, NULL, 'h'}, //hostPort @@ -896,9 +909,17 @@ int main(int argc, char **argv) { #endif #ifdef THREADED + if (batchMode) { + processline(cmd); + } while(!shutdownThisThing) { - int rc; - int len = sizeof(buffer) - bufoff -1; + int rc, len; + if (batchMode) { + // We are just waiting for the asynchronous command to complete. + millisleep(10); + continue; + } + len = sizeof(buffer) - bufoff -1; if (len <= 0) { fprintf(stderr, "Can't handle lines that long!\n"); exit(2); diff --git a/zookeeper-client/zookeeper-client-c/src/mt_adaptor.c b/zookeeper-client/zookeeper-client-c/src/mt_adaptor.c index 38cced425ba..668c2a09676 100644 --- a/zookeeper-client/zookeeper-client-c/src/mt_adaptor.c +++ b/zookeeper-client/zookeeper-client-c/src/mt_adaptor.c @@ -256,12 +256,13 @@ int adaptor_init(zhandle_t *zh) pthread_mutex_init(&zh->to_process.lock,0); pthread_mutex_init(&adaptor_threads->zh_lock,0); pthread_mutex_init(&adaptor_threads->reconfig_lock,0); - // to_send must be recursive mutex + pthread_mutex_init(&adaptor_threads->watchers_lock,0); + // to_send must be recursive mutex pthread_mutexattr_init(&recursive_mx_attr); pthread_mutexattr_settype(&recursive_mx_attr, PTHREAD_MUTEX_RECURSIVE); pthread_mutex_init(&zh->to_send.lock,&recursive_mx_attr); pthread_mutexattr_destroy(&recursive_mx_attr); - + pthread_mutex_init(&zh->sent_requests.lock,0); pthread_cond_init(&zh->sent_requests.cond,0); pthread_mutex_init(&zh->completions_to_process.lock,0); @@ -530,6 +531,25 @@ int unlock_reconfig(struct _zhandle *zh) } } +int lock_watchers(struct _zhandle *zh) +{ + struct adaptor_threads *adaptor = zh->adaptor_priv; + if (adaptor) { + return pthread_mutex_lock(&adaptor->watchers_lock); + } else { + return 0; + } +} +int unlock_watchers(struct _zhandle *zh) +{ + struct adaptor_threads *adaptor = zh->adaptor_priv; + if (adaptor) { + return pthread_mutex_unlock(&adaptor->watchers_lock); + } else { + return 0; + } +} + int enter_critical(zhandle_t* zh) { struct adaptor_threads *adaptor = zh->adaptor_priv; diff --git a/zookeeper-client/zookeeper-client-c/src/st_adaptor.c b/zookeeper-client/zookeeper-client-c/src/st_adaptor.c index ddc2582db34..07540b76ac1 100644 --- a/zookeeper-client/zookeeper-client-c/src/st_adaptor.c +++ b/zookeeper-client/zookeeper-client-c/src/st_adaptor.c @@ -94,6 +94,16 @@ int unlock_reconfig(struct _zhandle *zh) return 0; } +int lock_watchers(struct _zhandle *zh) +{ + return 0; +} + +int unlock_watchers(struct _zhandle *zh) +{ + return 0; +} + int enter_critical(zhandle_t* zh) { return 0; diff --git a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h index 8157472dd4b..305efbd168d 100644 --- a/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h +++ b/zookeeper-client/zookeeper-client-c/src/zk_adaptor.h @@ -166,6 +166,7 @@ struct adaptor_threads { pthread_mutex_t lock; // ... and a lock pthread_mutex_t zh_lock; // critical section lock pthread_mutex_t reconfig_lock; // lock for reconfiguring cluster's ensemble + pthread_mutex_t watchers_lock; // lock for watcher operations #ifdef WIN32 SOCKET self_pipe[2]; #else @@ -290,6 +291,10 @@ int zoo_unlock_auth(zhandle_t *zh); int lock_reconfig(struct _zhandle *zh); int unlock_reconfig(struct _zhandle *zh); +// watchers hashtable lock +int lock_watchers(struct _zhandle *zh); +int unlock_watchers(struct _zhandle *zh); + // critical section guards int enter_critical(zhandle_t* zh); int leave_critical(zhandle_t* zh); diff --git a/zookeeper-client/zookeeper-client-c/src/zookeeper.c b/zookeeper-client/zookeeper-client-c/src/zookeeper.c index 7ab5eed33cb..092ec932cfa 100644 --- a/zookeeper-client/zookeeper-client-c/src/zookeeper.c +++ b/zookeeper-client/zookeeper-client-c/src/zookeeper.c @@ -2002,9 +2002,11 @@ static int send_set_watches(zhandle_t *zh) int rc; req.relativeZxid = zh->last_zxid; + lock_watchers(zh); req.dataWatches.data = collect_keys(zh->active_node_watchers, (int*)&req.dataWatches.count); req.existWatches.data = collect_keys(zh->active_exist_watchers, (int*)&req.existWatches.count); req.childWatches.data = collect_keys(zh->active_child_watchers, (int*)&req.childWatches.count); + unlock_watchers(zh); // return if there are no pending watches if (!req.dataWatches.count && !req.existWatches.count && @@ -2850,7 +2852,9 @@ static int queue_session_event(zhandle_t *zh, int state) } /* We queued the buffer, so don't free it */ close_buffer_oarchive(&oa, 0); + lock_watchers(zh); cptr->c.watcher_result = collectWatchers(zh, ZOO_SESSION_EVENT, ""); + unlock_watchers(zh); queue_completion(&zh->completions_to_process, cptr, 0); if (process_async(zh->outstanding_sync)) { process_completions(zh); @@ -3156,7 +3160,9 @@ int zookeeper_process(zhandle_t *zh, int events) /* We are doing a notification, so there is no pending request */ c = create_completion_entry(zh, WATCHER_EVENT_XID,-1,0,0,0,0); c->buffer = bptr; + lock_watchers(zh); c->c.watcher_result = collectWatchers(zh, type, path); + unlock_watchers(zh); // We cannot free until now, otherwise path will become invalid deallocate_WatcherEvent(&evt); @@ -3211,8 +3217,10 @@ int zookeeper_process(zhandle_t *zh, int events) // Update last_zxid only when it is a request response zh->last_zxid = hdr.zxid; } + lock_watchers(zh); activateWatcher(zh, cptr->watcher, rc); deactivateWatcher(zh, cptr->watcher_deregistration, rc); + unlock_watchers(zh); if (cptr->c.void_result != SYNCHRONOUS_MARKER) { LOG_DEBUG(LOGCALLBACK(zh), "Queueing asynchronous response"); @@ -3495,6 +3503,49 @@ static int add_multi_completion(zhandle_t *zh, int xid, void_completion_t dc, return add_completion(zh, xid, COMPLETION_MULTI, dc, data, 0,0, clist); } +/** + * After sending the close request, we are waiting for a given millisecs for + * getting the answer and/or for the socket to be closed by the server. + * + * This function should not be called while we still want to process + * any response from the server. It must be called after adaptor_finish called, + * in order not to mess with the I/O receiver thread in multi-threaded mode. + */ +int wait_for_session_to_be_closed(zhandle_t *zh, int timeout_ms) +{ + int ret = 0; +#ifndef WIN32 + struct pollfd fd_s[1]; +#else + fd_set rfds; + struct timeval waittime = {timeout_ms / 1000, (timeout_ms % 1000) * 1000}; +#endif + + if (zh == NULL) { + return ZBADARGUMENTS; + } + +#ifndef WIN32 + fd_s[0].fd = zh->fd->sock; + fd_s[0].events = POLLIN; + ret = poll(fd_s, 1, timeout_ms); +#else + FD_ZERO(&rfds); + FD_SET(zh->fd->sock , &rfds); + ret = select(zh->fd->sock + 1, &rfds, NULL, NULL, &waittime); +#endif + + if (ret == 0){ + LOG_WARN(LOGCALLBACK(zh), "Timed out (%dms) during waiting for server's reply after sending a close request, sessionId=%#llx\n", + timeout_ms, zh->client_id.client_id); + } else if (ret < 0) { + LOG_WARN(LOGCALLBACK(zh), "System error (%d) happened while waiting for server's reply, sessionId=%#llx\n", + ret, zh->client_id.client_id); + } + + return ZOK; +} + int zookeeper_close(zhandle_t *zh) { int rc=ZOK; @@ -3520,32 +3571,33 @@ int zookeeper_close(zhandle_t *zh) } /* No need to decrement the counter since we're just going to * destroy the handle later. */ - if (is_connected(zh)){ + if (is_connected(zh)) { struct oarchive *oa; struct RequestHeader h = {get_xid(), ZOO_CLOSE_OP}; LOG_INFO(LOGCALLBACK(zh), "Closing zookeeper sessionId=%#llx to %s\n", - zh->client_id.client_id,zoo_get_current_server(zh)); + zh->client_id.client_id, zoo_get_current_server(zh)); oa = create_buffer_oarchive(); rc = serialize_RequestHeader(oa, "header", &h); - rc = rc < 0 ? rc : queue_buffer_bytes(&zh->to_send, get_buffer(oa), - get_buffer_len(oa)); + rc = rc < 0 ? rc : queue_buffer_bytes(&zh->to_send, get_buffer(oa), get_buffer_len(oa)); /* We queued the buffer, so don't free it */ close_buffer_oarchive(&oa, 0); if (rc < 0) { + LOG_DEBUG(LOGCALLBACK(zh), "Error during closing zookeeper session, sessionId=%#llx to %s (error: %d)\n", + zh->client_id.client_id, zoo_get_current_server(zh), rc); rc = ZMARSHALLINGERROR; - goto finish; - } + } else { + /* make sure the close request is sent; we set timeout to an arbitrary + * (but reasonable) number of milliseconds since we want the call to block*/ + rc = adaptor_send_queue(zh, 3000); - /* make sure the close request is sent; we set timeout to an arbitrary - * (but reasonable) number of milliseconds since we want the call to block*/ - rc=adaptor_send_queue(zh, 3000); - }else{ - LOG_INFO(LOGCALLBACK(zh), "Freeing zookeeper resources for sessionId=%#llx\n", - zh->client_id.client_id); + /* give some time to the server to process the session close request properly */ + rc = rc < 0 ? rc : wait_for_session_to_be_closed(zh, 1500); + } + } else { rc = ZOK; } -finish: + LOG_INFO(LOGCALLBACK(zh), "Freeing zookeeper resources for sessionId=%#llx\n", zh->client_id.client_id); destroy(zh); adaptor_destroy(zh); free(zh->fd); @@ -4459,19 +4511,23 @@ static int aremove_watches( goto done; } + lock_watchers(zh); if (!pathHasWatcher(zh, server_path, wtype, watcher, watcherCtx)) { rc = ZNOWATCHER; + unlock_watchers(zh); goto done; } if (local) { removeWatchers(zh, server_path, wtype, watcher, watcherCtx); + unlock_watchers(zh); #ifdef THREADED notify_sync_completion((struct sync_completion *)data); #endif rc = ZOK; goto done; } + unlock_watchers(zh); oa = create_buffer_oarchive(); rc = serialize_RequestHeader(oa, "header", &h); @@ -4763,7 +4819,7 @@ int zoo_add_auth(zhandle_t *zh,const char* scheme,const char* cert, static const char* format_endpoint_info(const struct sockaddr_storage* ep) { - static char buf[128] = { 0 }; + static char buf[134] = { 0 }; char addrstr[INET6_ADDRSTRLEN] = { 0 }; const char *fmtstring; void *inaddr; diff --git a/zookeeper-client/zookeeper-client-c/tests/TestReconfig.cc b/zookeeper-client/zookeeper-client-c/tests/TestReconfig.cc index 317ffcddfed..22f87e94491 100644 --- a/zookeeper-client/zookeeper-client-c/tests/TestReconfig.cc +++ b/zookeeper-client/zookeeper-client-c/tests/TestReconfig.cc @@ -26,6 +26,10 @@ #include #include +extern "C" { +#include +} + #include "Util.h" #include "LibCMocks.h" #include "ZKMocks.h" @@ -218,6 +222,10 @@ class Zookeeper_reconfig : public CPPUNIT_NS::TestFixture CPPUNIT_TEST(testcycleNextServer); CPPUNIT_TEST(testMigrateOrNot); CPPUNIT_TEST(testMigrationCycle); + CPPUNIT_TEST(testAddrVecContainsIPv4); +#ifdef AF_INET6 + CPPUNIT_TEST(testAddrVecContainsIPv6); +#endif // In threaded mode each 'create' is a thread -- it's not practical to create // 10,000 threads to test load balancing. The load balancing code can easily @@ -609,6 +617,81 @@ class Zookeeper_reconfig : public CPPUNIT_NS::TestFixture numServers = 9; updateAllClientsAndServers(numServers); } + + /** + * This tests that client can detect server's ipv4 address change. + * + * (1) We generate some address and put in addr, which saddr point to + * (2) Add all addresses that differ by one bit from the source + * (3) Add same address, but set ipv6 protocol + * (4) Ensure, that our address is not equal to any of generated, + * and that it equals to itself + */ + void testAddrVecContainsIPv4() { + addrvec_t vec; + addrvec_init(&vec); + + sockaddr_storage addr; + sockaddr_in* saddr = (sockaddr_in*)&addr; + saddr->sin_family = AF_INET; + saddr->sin_port = htons((u_short)1234); + saddr->sin_addr.s_addr = INADDR_ANY; + + CPPUNIT_ASSERT(sizeof(saddr->sin_addr.s_addr) == 4); + + for (int i = 0; i < 32; i++) { + saddr->sin_addr.s_addr ^= (1 << i); + addrvec_append(&vec, &addr); + saddr->sin_addr.s_addr ^= (1 << i); + } + + saddr->sin_family = AF_INET6; + addrvec_append(&vec, &addr); + saddr->sin_family = AF_INET; + + CPPUNIT_ASSERT(!addrvec_contains(&vec, &addr)); + addrvec_append(&vec, &addr); + CPPUNIT_ASSERT(addrvec_contains(&vec, &addr)); + addrvec_free(&vec); + } + + /** + * This tests that client can detect server's ipv6 address change. + * + * Same logic as in previous testAddrVecContainsIPv4 method, + * but we keep in mind, that ipv6 is 128-bit long. + */ +#ifdef AF_INET6 + void testAddrVecContainsIPv6() { + addrvec_t vec; + addrvec_init(&vec); + + sockaddr_storage addr; + sockaddr_in6* saddr = (sockaddr_in6*)&addr; + saddr->sin6_family = AF_INET6; + saddr->sin6_port = htons((u_short)1234); + saddr->sin6_addr = in6addr_any; + + CPPUNIT_ASSERT(sizeof(saddr->sin6_addr.s6_addr) == 16); + + for (int i = 0; i < 16; i++) { + for (int j = 0; j < 8; j++) { + saddr->sin6_addr.s6_addr[i] ^= (1 << j); + addrvec_append(&vec, &addr); + saddr->sin6_addr.s6_addr[i] ^= (1 << j); + } + } + + saddr->sin6_family = AF_INET; + addrvec_append(&vec, &addr); + saddr->sin6_family = AF_INET6; + + CPPUNIT_ASSERT(!addrvec_contains(&vec, &addr)); + addrvec_append(&vec, &addr); + CPPUNIT_ASSERT(addrvec_contains(&vec, &addr)); + addrvec_free(&vec); + } +#endif }; CPPUNIT_TEST_SUITE_REGISTRATION(Zookeeper_reconfig); diff --git a/zookeeper-compatibility-tests/pom.xml b/zookeeper-compatibility-tests/pom.xml new file mode 100644 index 00000000000..dc4a97295a6 --- /dev/null +++ b/zookeeper-compatibility-tests/pom.xml @@ -0,0 +1,58 @@ + + + + + com.linkedin.zookeeper + parent + 3.6.2 + + pom + 4.0.0 + + zookeeper-compatibility-tests + + + zookeeper-compatibility-tests-curator + + + Apache ZooKeeper - Compatibility Tests + Module for various compatibility tests + + + + + org.apache.maven.plugins + maven-install-plugin + + true + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + diff --git a/zookeeper-compatibility-tests/zookeeper-compatibility-tests-curator/pom.xml b/zookeeper-compatibility-tests/zookeeper-compatibility-tests-curator/pom.xml new file mode 100644 index 00000000000..889f9a0ea54 --- /dev/null +++ b/zookeeper-compatibility-tests/zookeeper-compatibility-tests-curator/pom.xml @@ -0,0 +1,77 @@ + + + + + com.linkedin.zookeeper + zookeeper-compatibility-tests + 3.6.2 + + 4.0.0 + + zookeeper-compatibility-tests-curator + + Apache ZooKeeper - Compatibility Tests - Curator + Module for Apache Curator compatibility tests + + + 5.0.0 + + + + + com.linkedin.zookeeper + zookeeper + ${project.version} + + + + org.apache.curator + curator-recipes + ${apache-curator-version} + + + com.linkedin.zookeeper + zookeeper + + + test + + + + org.apache.curator + curator-test + ${apache-curator-version} + + + com.linkedin.zookeeper + zookeeper + + + test + + + + junit + junit + test + + + \ No newline at end of file diff --git a/zookeeper-compatibility-tests/zookeeper-compatibility-tests-curator/src/test/java/org/apache/zookeeper/compatibility/TestApacheCuratorCompatibility.java b/zookeeper-compatibility-tests/zookeeper-compatibility-tests-curator/src/test/java/org/apache/zookeeper/compatibility/TestApacheCuratorCompatibility.java new file mode 100644 index 00000000000..b60a0042244 --- /dev/null +++ b/zookeeper-compatibility-tests/zookeeper-compatibility-tests-curator/src/test/java/org/apache/zookeeper/compatibility/TestApacheCuratorCompatibility.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.compatibility; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.recipes.cache.CuratorCache; +import org.apache.curator.retry.RetryOneTime; +import org.apache.curator.test.TestingServer; +import org.junit.Test; + +/** + * Make sure minimal Apache Curator APIs work correctly. As it's a widely used ZooKeeper + * client library we should not break it. + */ +public class TestApacheCuratorCompatibility { + private static final int TIMEOUT_MS = 5000; + + @Test + public void testBasicUsageOfApisAndRecipes() throws Exception { + try (TestingServer server = new TestingServer()) { + RetryOneTime retryPolicy = new RetryOneTime(1); + try (CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(), retryPolicy)) { + try (CuratorCache cache = CuratorCache.build(client, "/base/path")) { + client.start(); + cache.start(); + + BlockingQueue paths = new LinkedBlockingQueue<>(); + cache.listenable().addListener((dummy1, dummy2, data) -> paths.add(data.getPath())); + + client.create().creatingParentsIfNeeded().forPath("/base/path/1"); + client.create().creatingParentsIfNeeded().forPath("/base/path/2"); + client.create().creatingParentsIfNeeded().forPath("/base/path/1/a"); + client.create().creatingParentsIfNeeded().forPath("/base/path/2/a"); + + assertEquals("/base/path", poll(paths)); + assertEquals("/base/path/1", poll(paths)); + assertEquals("/base/path/2", poll(paths)); + assertEquals("/base/path/1/a", poll(paths)); + assertEquals("/base/path/2/a", poll(paths)); + } + } + } + } + + private static String poll(BlockingQueue queue) { + try { + String value = queue.poll(TIMEOUT_MS, TimeUnit.MILLISECONDS); + assertNotNull("Event poll timed out", value); + return value; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } +} diff --git a/zookeeper-contrib/build-contrib.xml b/zookeeper-contrib/build-contrib.xml index 70bb8644f52..34f3c9b03b3 100644 --- a/zookeeper-contrib/build-contrib.xml +++ b/zookeeper-contrib/build-contrib.xml @@ -32,7 +32,7 @@ - + @@ -47,7 +47,7 @@ - + @@ -179,7 +179,7 @@ - + @@ -190,7 +190,7 @@ - diff --git a/zookeeper-contrib/ivysettings.xml b/zookeeper-contrib/ivysettings.xml new file mode 100644 index 00000000000..15c98b6ec4e --- /dev/null +++ b/zookeeper-contrib/ivysettings.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/zookeeper-contrib/pom.xml b/zookeeper-contrib/pom.xml index 075363f3cfb..2a52e06da58 100755 --- a/zookeeper-contrib/pom.xml +++ b/zookeeper-contrib/pom.xml @@ -22,12 +22,11 @@ 4.0.0 - org.apache.zookeeper + com.linkedin.zookeeper parent - 3.6.0-SNAPSHOT + 3.6.2 - org.apache.zookeeper zookeeper-contrib pom Apache ZooKeeper - Contrib @@ -35,24 +34,31 @@ Contrib projects to Apache ZooKeeper - - zookeeper-contrib-loggraph - zookeeper-contrib-rest - zookeeper-contrib-zooinspector - + + true + + + + + fatjar + + zookeeper-contrib-fatjar + + + + full-build + + zookeeper-contrib-fatjar + zookeeper-contrib-loggraph + zookeeper-contrib-rest + zookeeper-contrib-zooinspector + + + - - com.github.spotbugs - spotbugs-maven-plugin - 3.1.9 - - - true - - org.apache.maven.plugins maven-compiler-plugin diff --git a/zookeeper-contrib/zookeeper-contrib-fatjar/README.md b/zookeeper-contrib/zookeeper-contrib-fatjar/README.md new file mode 100644 index 00000000000..baf2f95f91f --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-fatjar/README.md @@ -0,0 +1,21 @@ +ZooKeeper Fatjar +================ + +This package contains build to create a fat zookeeper jar. Fatjar can be used to run: +- zookeeper server +- zookeeper client +- distributed load generator for testing (generateLoad) +- container that will instantiate classes as directed by an instance manager (ic) +- system test (systest) +- jmh micro benchmarks (jmh) + + +Use following command to build fatjar +``` +mvn clean install -P fatjar -DskipTests +``` + +To run the fatjar use: +``` +java -jar zoookeeper--fatjar.jar +``` diff --git a/zookeeper-contrib/zookeeper-contrib-fatjar/README.txt b/zookeeper-contrib/zookeeper-contrib-fatjar/README.txt deleted file mode 100644 index f8027ae8c9e..00000000000 --- a/zookeeper-contrib/zookeeper-contrib-fatjar/README.txt +++ /dev/null @@ -1,2 +0,0 @@ -This package contains build to create a fat zookeeper jar. You need to run ant to create the fat jar. -To run the fatjar you can use. java -jar zoookeeper-*fatjar.jar diff --git a/zookeeper-contrib/zookeeper-contrib-fatjar/build.xml b/zookeeper-contrib/zookeeper-contrib-fatjar/build.xml deleted file mode 100644 index 3a1dd958e4e..00000000000 --- a/zookeeper-contrib/zookeeper-contrib-fatjar/build.xml +++ /dev/null @@ -1,74 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/zookeeper-contrib/zookeeper-contrib-fatjar/pom.xml b/zookeeper-contrib/zookeeper-contrib-fatjar/pom.xml new file mode 100755 index 00000000000..58343acb20e --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-fatjar/pom.xml @@ -0,0 +1,145 @@ + + + + 4.0.0 + + com.linkedin.zookeeper + zookeeper-contrib + 3.6.2 + + + com.linkedin.zookeeper + zookeeper-contrib-fatjar + jar + Apache ZooKeeper - Contrib - Fatjar + + + true + true + + + + + com.linkedin.zookeeper + zookeeper-jute + ${project.version} + + + com.linkedin.zookeeper + zookeeper + ${project.version} + + + com.linkedin.zookeeper + zookeeper + ${project.version} + test-jar + + + com.linkedin.zookeeper + zookeeper-it + ${project.version} + + + org.slf4j + slf4j-api + + + commons-cli + commons-cli + + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-servlet + + + com.fasterxml.jackson.core + jackson-databind + + + com.googlecode.json-simple + json-simple + + + jline + jline + + + io.dropwizard.metrics + metrics-core + + + org.xerial.snappy + snappy-java + + + log4j + log4j + + + + + + + ${project.basedir}/src/main/resources + + + ${project.basedir}/../../conf + + log4j.properties + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + jar-with-dependencies + + + + org.apache.zookeeper.util.FatJarMain + + + zookeeper-${project.version}-fatjar + false + false + + + + make-assembly + package + + single + + + + + + + + diff --git a/zookeeper-contrib/zookeeper-contrib-fatjar/src/main/resources/mainClasses b/zookeeper-contrib/zookeeper-contrib-fatjar/src/main/resources/mainClasses index ba29e891973..c7b27a1325b 100644 --- a/zookeeper-contrib/zookeeper-contrib-fatjar/src/main/resources/mainClasses +++ b/zookeeper-contrib/zookeeper-contrib-fatjar/src/main/resources/mainClasses @@ -4,8 +4,6 @@ client:org.apache.zookeeper.ZooKeeperMain:Client shell to ZooKeeper server:org.apache.zookeeper.server.quorum.QuorumPeerMain:Start ZooKeeper server ::Test Commands generateLoad:org.apache.zookeeper.test.system.GenerateLoad:A distributed load generator for testing -quorumBench:org.apache.zookeeper.server.QuorumBenchmark:A benchmark of just the quorum protocol -abBench:org.apache.zookeeper.server.quorum.AtomicBroadcastBenchmark:A benchmark of just the atomic broadcast ic:org.apache.zookeeper.test.system.InstanceContainer:A container that will instantiate classes as directed by an instance manager systest:org.apache.zookeeper.test.system.BaseSysTest:Start system test jmh:org.apache.zookeeper.BenchMain:Run jmh micro benchmarks diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/pom.xml b/zookeeper-contrib/zookeeper-contrib-loggraph/pom.xml index 5e9449538e4..a7300432964 100755 --- a/zookeeper-contrib/zookeeper-contrib-loggraph/pom.xml +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/pom.xml @@ -21,12 +21,11 @@ --> 4.0.0 - org.apache.zookeeper + com.linkedin.zookeeper zookeeper-contrib - 3.6.0-SNAPSHOT + 3.6.2 - org.apache.zookeeper zookeeper-contrib-loggraph jar Apache ZooKeeper - Contrib - Loggraph @@ -36,12 +35,12 @@ - org.apache.zookeeper + com.linkedin.zookeeper zookeeper-jute ${project.version} - org.apache.zookeeper + com.linkedin.zookeeper zookeeper ${project.version} diff --git a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/TxnLogSource.java b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/TxnLogSource.java index 809c4551390..ad2e2589376 100644 --- a/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/TxnLogSource.java +++ b/zookeeper-contrib/zookeeper-contrib-loggraph/src/main/java/org/apache/zookeeper/graph/TxnLogSource.java @@ -31,6 +31,7 @@ import org.apache.jute.InputArchive; import org.apache.jute.Record; import org.apache.zookeeper.server.TraceFormatter; +import org.apache.zookeeper.server.TxnLogEntry; import org.apache.zookeeper.server.persistence.FileHeader; import org.apache.zookeeper.server.persistence.FileTxnLog; import org.apache.zookeeper.server.util.SerializeUtils; @@ -180,8 +181,10 @@ private LogEntry readNextEntry() { throw new IOException("CRC doesn't match " + crcValue + " vs " + crc.getValue()); } - TxnHeader hdr = new TxnHeader(); - Record r = SerializeUtils.deserializeTxn(bytes, hdr); + + TxnLogEntry logEntry = SerializeUtils.deserializeTxn(bytes); + TxnHeader hdr = logEntry.getHeader(); + Record r = logEntry.getTxn(); switch (hdr.getType()) { case OpCode.createSession: { @@ -327,8 +330,9 @@ public TxnLogSource(String file) throws IOException { if (logStream.readByte("EOR") != 'B') { throw new EOFException("Last transaction was partial."); } - TxnHeader hdr = new TxnHeader(); - Record r = SerializeUtils.deserializeTxn(bytes, hdr); + TxnLogEntry logEntry = SerializeUtils.deserializeTxn(bytes); + TxnHeader hdr = logEntry.getHeader(); + Record r = logEntry.getTxn(); if (starttime == 0) { starttime = hdr.getTime(); diff --git a/zookeeper-contrib/zookeeper-contrib-rest/pom.xml b/zookeeper-contrib/zookeeper-contrib-rest/pom.xml index aa9ea4dabd2..83ecbec7c90 100755 --- a/zookeeper-contrib/zookeeper-contrib-rest/pom.xml +++ b/zookeeper-contrib/zookeeper-contrib-rest/pom.xml @@ -21,12 +21,11 @@ --> 4.0.0 - org.apache.zookeeper + com.linkedin.zookeeper zookeeper-contrib - 3.6.0-SNAPSHOT + 3.6.2 - org.apache.zookeeper zookeeper-contrib-rest jar Apache ZooKeeper - Contrib - Rest @@ -54,12 +53,12 @@ - org.apache.zookeeper + com.linkedin.zookeeper zookeeper ${project.version} - org.apache.zookeeper + com.linkedin.zookeeper zookeeper ${project.version} test-jar diff --git a/zookeeper-contrib/zookeeper-contrib-zkpython/README b/zookeeper-contrib/zookeeper-contrib-zkpython/README index 89d99989756..ffad255f06f 100644 --- a/zookeeper-contrib/zookeeper-contrib-zkpython/README +++ b/zookeeper-contrib/zookeeper-contrib-zkpython/README @@ -5,20 +5,26 @@ Please do not rely on APIs staying constant in the short term. The handling of e DEPENDENCIES: ------------- -This has only been tested against SVN (i.e. 3.2.0 in development) but should work against 3.1.1. +This has only been tested against SVN/Git (i.e. 3.2.0 in development) but should work against 3.1.1. -You will need the Python development headers installed to build the module - on many package-management systems, these can be found in python-devel. +You will need the Python development headers installed to build the module - on many package-management systems, these can be found in python-devel. (On ubuntu 18.4, install python2.7 and python2.7-dev.) -Python >= 2.6 is required. We have tested against 2.6. We have not tested against 3.x. +Python >= 2.6 is required. We have tested against 2.6 and 3.5+. + +By default, the extension assumes that the C client library was compiled with OpenSSL enabled (--with-openssl). You can disable OpenSSL support in the Python binding by setting the ZKPYTHON_NO_SSL environment variable to a non-empty string before executing Ant or setup.py. + +E.g. setting up python and python devel on ubuntu 18.4: +sudo apt-get install python2.7 python2.7-dev +sudo update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 BUILD AND INSTALL: ------------------- -To install, make sure that the C client has been built and that the libraries are installed in /usr/local/lib (or change this directory in setup.py). Then run: +To install, make sure that the C client has been built (use `mvn clean install -DskipTests -Pfull-build` in the root folder of zookeeper) or that the zookeeper C libraries are installed in /usr/local/lib (or change this directory in setup.py). Then run: ant install -from zookeeper/src/contrib/zkpython/. +from zookeeper-contrib/zookeeper-contrib-zkpython/. To test, run ant test from the same directory. diff --git a/zookeeper-contrib/zookeeper-contrib-zkpython/build.xml b/zookeeper-contrib/zookeeper-contrib-zkpython/build.xml index c229deec654..99f634abe81 100644 --- a/zookeeper-contrib/zookeeper-contrib-zkpython/build.xml +++ b/zookeeper-contrib/zookeeper-contrib-zkpython/build.xml @@ -20,10 +20,10 @@ - + = 3 + Py_DECREF(module); + return PyErr_NoMemory(); +#else + return; +#endif } ZooKeeperException = PyErr_NewException("zookeeper.ZooKeeperException", diff --git a/zookeeper-contrib/zookeeper-contrib-zkpython/src/python/setup.py b/zookeeper-contrib/zookeeper-contrib-zkpython/src/python/setup.py index c64889e0312..b225a317400 100755 --- a/zookeeper-contrib/zookeeper-contrib-zkpython/src/python/setup.py +++ b/zookeeper-contrib/zookeeper-contrib-zkpython/src/python/setup.py @@ -15,17 +15,26 @@ # limitations under the License. from distutils.core import setup, Extension +import os zookeeper_basedir = "../../" +zookeeper_macros = [("THREADED", None)] + +# Assume the C extension includes OpenSSL support unless told +# otherwise. +if not os.environ.get("ZKPYTHON_NO_SSL"): + zookeeper_macros.append(("HAVE_OPENSSL_H", True)) + zookeepermodule = Extension("zookeeper", sources=["src/c/zookeeper.c"], + define_macros=zookeeper_macros, include_dirs=[zookeeper_basedir + "/zookeeper-client/zookeeper-client-c/include", - zookeeper_basedir + "/build/c", + zookeeper_basedir + "/zookeeper-client/zookeeper-client-c/target/c", zookeeper_basedir + "/zookeeper-client/zookeeper-client-c/generated"], libraries=["zookeeper_mt"], library_dirs=[zookeeper_basedir + "/zookeeper-client/zookeeper-client-c/.libs/", - zookeeper_basedir + "/build/c/.libs/", + zookeeper_basedir + "/zookeeper-client/zookeeper-client-c/target/c/.libs/", zookeeper_basedir + "/build/test/test-cppunit/.libs", "/usr/local/lib" ]) diff --git a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/async_test.py b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/async_test.py index e81343570ea..61740ae433f 100644 --- a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/async_test.py +++ b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/async_test.py @@ -26,7 +26,7 @@ def setUp( self ): def test_async(self): self.assertEqual(self.connected, True) - ret = zookeeper.async(self.handle, "/") + ret = getattr(zookeeper, 'async')(self.handle, "/") self.assertEqual(ret, zookeeper.OK, "async failed") if __name__ == '__main__': diff --git a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/callback_test.py b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/callback_test.py index 55d7fe17866..95e20b4dea1 100644 --- a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/callback_test.py +++ b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/callback_test.py @@ -91,9 +91,9 @@ def dispatch_callback(*args, **kwargs): self.create_callback( dispatch_callback )), lambda: self.assertEqual(True, self.callback_flag, "Strings dispatch not fired")) - self.callback_harness( lambda: zookeeper.async(self.handle, - "/", - self.create_callback( dispatch_callback )), + self.callback_harness( lambda: getattr(zookeeper, 'async')(self.handle, + "/", + self.create_callback( dispatch_callback )), lambda: self.assertEqual(True, self.callback_flag, "String dispatch not fired")) self.callback_harness( lambda: zookeeper.aget_acl(self.handle, diff --git a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/connection_test.py b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/connection_test.py index 3913fe3b7d9..3fbbd4bf85b 100755 --- a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/connection_test.py +++ b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/connection_test.py @@ -58,6 +58,38 @@ def connection_watcher(handle, type, state, path): self.handle, "/") + @unittest.skipUnless(hasattr(zookeeper, 'init_ssl'), + "SSL support not compiled in.") + def testsslconnection(self): + cv = threading.Condition() + self.connected = False + def connection_watcher(handle, type, state, path): + cv.acquire() + self.connected = True + self.assertEqual(zookeeper.CONNECTED_STATE, state) + self.handle = handle + cv.notify() + cv.release() + + cv.acquire() + ret = zookeeper.init_ssl(self.sslhost, self.sslcert, connection_watcher) + cv.wait(15.0) + cv.release() + self.assertEqual(self.connected, True, "SSL Connection timed out to " + self.host) + self.assertEqual(zookeeper.CONNECTED_STATE, zookeeper.state(self.handle)) + + self.assertEqual(zookeeper.close(self.handle), zookeeper.OK) + # Trying to close the same handle twice is an error, and the C library will segfault on it + # so make sure this is caught at the Python module layer + self.assertRaises(zookeeper.ZooKeeperException, + zookeeper.close, + self.handle) + + self.assertRaises(zookeeper.ZooKeeperException, + zookeeper.get, + self.handle, + "/") + def testhandlereuse(self): """ Test a) multiple concurrent connections b) reuse of closed handles diff --git a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/run_tests.sh b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/run_tests.sh index 91e5b57d2f0..232359b16e0 100755 --- a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/run_tests.sh +++ b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/run_tests.sh @@ -19,6 +19,8 @@ # Usage: run_tests.sh testdir [logdir] # logdir is optional, defaults to cwd +set -e + # get the number of command-line arguments given ARGC=$# @@ -30,11 +32,12 @@ else fi # Find the build directory containing zookeeper.so -SO_PATH=`find ../../build/ -name "zookeeper.so" | head -1` +SO_PATH=`find ./target/ -name 'zookeeper*.so' | head -1` PYTHONPATH=`dirname $SO_PATH` -LIB_PATH=../../build/c/.libs/:../../build/test/test-cppunit/.libs +LIB_PATH=../../zookeeper-client/zookeeper-client-c/target/c/.libs for test in `ls $1/*_test.py`; do echo "Running $test" + echo "Running LD_LIBRARY_PATH=$LIB_PATH:$LD_LIBRARY_PATH DYLD_LIBRARY_PATH=$LIB_PATH:$DYLD_LIBRARY_PATH PYTHONPATH=$PYTHONPATH python $test" LD_LIBRARY_PATH=$LIB_PATH:$LD_LIBRARY_PATH DYLD_LIBRARY_PATH=$LIB_PATH:$DYLD_LIBRARY_PATH PYTHONPATH=$PYTHONPATH python $test done diff --git a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zkServer.sh b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zkServer.sh index a4638de3f31..3b6ed37fc12 100755 --- a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zkServer.sh +++ b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zkServer.sh @@ -18,36 +18,56 @@ if [ "x$1" == "x" ] then - echo "USAGE: $0 startClean|start|stop hostPorts" + echo "USAGE: $0 startClean|start|stop" exit 2 fi -if [ "x$1" == "xstartClean" ] +if [ "x${base_dir}" == "x" ] +then + PROJECT_ROOT="../../" +else + PROJECT_ROOT=${base_dir} +fi +WORK_DIR=${PROJECT_ROOT}/zookeeper-contrib/zookeeper-contrib-zkpython/target/zkpython_tests +TEST_DIR=${PROJECT_ROOT}/zookeeper-contrib/zookeeper-contrib-zkpython/src/test + + +if [ -r "${WORK_DIR}/../zk.pid" ] then - if [ "x${base_dir}" == "x" ] + pid=`cat "${WORK_DIR}/../zk.pid"` + kill -9 $pid + rm -f "${WORK_DIR}/../zk.pid" +fi + +which lsof &> /dev/null +if [ $? -eq 0 ] +then + pid=`lsof -i :22182 | grep LISTEN | awk '{print $2}'` + if [ -n "$pid" ] then - rm -rf /tmp/zkdata - else - rm -rf ${base_dir}/build/tmp + kill -9 $pid fi fi -if [ "x${base_dir}" == "x" ] + + + +if [ "x$1" == "xstartClean" ] then -zk_base="../../" -else -zk_base="${base_dir}" + rm -rf ${WORK_DIR} fi -CLASSPATH="$CLASSPATH:${zk_base}/build/classes" + + +CLASSPATH="$CLASSPATH:${PROJECT_ROOT}/zookeeper-server/target/classes" CLASSPATH="$CLASSPATH:${zk_base}/conf" -for i in "${zk_base}"/build/lib/*.jar +for i in "${PROJECT_ROOT}"/zookeeper-server/target/lib/*.jar do CLASSPATH="$CLASSPATH:$i" done -for i in "${zk_base}"/zookeeper-server/src/main/resource/lib/*.jar +for i in "${PROJECT_ROOT}"/zookeeper-server/src/main/resource/lib/*.jar do CLASSPATH="$CLASSPATH:$i" done @@ -57,15 +77,20 @@ done case $1 in start|startClean) - if [ "x${base_dir}" == "x" ] - then - mkdir -p /tmp/zkdata - java -cp $CLASSPATH org.apache.zookeeper.server.ZooKeeperServerMain 22182 /tmp/zkdata &> /tmp/zk.log & - else - mkdir -p ${base_dir}/build/tmp/zkdata - java -cp $CLASSPATH org.apache.zookeeper.server.ZooKeeperServerMain 22182 ${base_dir}/build/tmp/zkdata &> ${base_dir}/build/tmp/zk.log & - fi - sleep 5 + mkdir -p ${WORK_DIR}/zkdata + + rm -rf ${WORK_DIR}/ssl + mkdir -p ${WORK_DIR}/ssl + cp ${PROJECT_ROOT}/zookeeper-client/zookeeper-client-c/ssl/gencerts.sh ${WORK_DIR}/ssl/ + cd ${WORK_DIR}/ssl/ + ./gencerts.sh + cd - + + sed "s#WORKDIR#${WORK_DIR}#g" ${TEST_DIR}/zoo.cfg > "${WORK_DIR}/zoo.cfg" + java -Dzookeeper.extendedTypesEnabled=true -Dznode.container.checkIntervalMs=100 -cp $CLASSPATH org.apache.zookeeper.server.ZooKeeperServerMain "${WORK_DIR}/zoo.cfg" &> "${WORK_DIR}/zoo.log" & + pid=$! + echo -n $! > ${WORK_DIR}/../zk.pid + sleep 5 ;; stop) # Already killed above diff --git a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zktestbase.py b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zktestbase.py index 8229418e3e7..cafbec56669 100755 --- a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zktestbase.py +++ b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zktestbase.py @@ -22,10 +22,13 @@ class TestBase(unittest.TestCase): SERVER_PORT = 22182 - + SERVER_SSL_PORT = 22183 + def __init__(self,methodName='runTest'): unittest.TestCase.__init__(self,methodName) self.host = "localhost:%d" % self.SERVER_PORT + self.sslhost = "localhost:%d" % self.SERVER_SSL_PORT + self.sslcert = "./target/zkpython_tests/ssl/server.crt,./target/zkpython_tests/ssl/client.crt,./target/zkpython_tests/ssl/clientkey.pem,password" self.connected = False self.handle = -1 logdir = os.environ.get("ZKPY_LOG_DIR") diff --git a/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zoo.cfg b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zoo.cfg new file mode 100644 index 00000000000..2d4fc31d9fd --- /dev/null +++ b/zookeeper-contrib/zookeeper-contrib-zkpython/src/test/zoo.cfg @@ -0,0 +1,14 @@ +tickTime=500 +initLimit=10 +syncLimit=5 +dataDir=WORKDIR/zkdata +maxClientCnxns=200 + +clientPort=22182 +secureClientPort=22183 +serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory +ssl.keyStore.location=WORKDIR/ssl/server.jks +ssl.keyStore.password=password +ssl.trustStore.location=WORKDIR/ssl/servertrust.jks +ssl.trustStore.password=password + diff --git a/zookeeper-contrib/zookeeper-contrib-zooinspector/pom.xml b/zookeeper-contrib/zookeeper-contrib-zooinspector/pom.xml index 87c52d91da9..f688f842d1a 100755 --- a/zookeeper-contrib/zookeeper-contrib-zooinspector/pom.xml +++ b/zookeeper-contrib/zookeeper-contrib-zooinspector/pom.xml @@ -21,12 +21,11 @@ --> 4.0.0 - org.apache.zookeeper + com.linkedin.zookeeper zookeeper-contrib - 3.6.0-SNAPSHOT + 3.6.2 - org.apache.zookeeper zookeeper-contrib-zooinspector jar Apache ZooKeeper - Contrib - ZooInspector @@ -42,7 +41,7 @@ - org.apache.zookeeper + com.linkedin.zookeeper zookeeper ${project.version} diff --git a/zookeeper-docs/pom.xml b/zookeeper-docs/pom.xml index eb56544de5d..259f9be22ff 100644 --- a/zookeeper-docs/pom.xml +++ b/zookeeper-docs/pom.xml @@ -21,13 +21,12 @@ --> 4.0.0 - org.apache.zookeeper + com.linkedin.zookeeper parent - 3.6.0-SNAPSHOT + 3.6.2 .. - org.apache.zookeeper zookeeper-docs Apache ZooKeeper - Documentation Documentation diff --git a/zookeeper-docs/src/main/resources/markdown/html/header.html b/zookeeper-docs/src/main/resources/markdown/html/header.html index ce82ecea565..c529f880747 100644 --- a/zookeeper-docs/src/main/resources/markdown/html/header.html +++ b/zookeeper-docs/src/main/resources/markdown/html/header.html @@ -19,9 +19,6 @@ Apache > ZooKeeper

- @@ -73,7 +70,7 @@