From 9fea6674fbcaead59e090c28dbd7a59e864c442e Mon Sep 17 00:00:00 2001 From: Powell Molleti Date: Sat, 13 Aug 2016 19:27:03 -0700 Subject: [PATCH 01/12] SSL support for ZAB and FLE. Code for dynamic reconfig() and SSL support. --- .gitignore | 18 + build.xml | 1 + conf/multi/README.txt | 4 + conf/multi/node1.log4j.properties | 58 +++ conf/multi/node2.log4j.properties | 58 +++ conf/multi/node3.log4j.properties | 58 +++ conf/multi/start_quorum.sh | 45 ++ conf/multi/zoo1.cfg | 5 + conf/multi/zoo2.cfg | 5 + conf/multi/zoo3.cfg | 5 + ivy.xml | 4 +- resources/README.txt | 17 + resources/gencert.sh | 53 +++ resources/init.sh | 39 ++ resources/x509ca/openssl-ca.cnf | 86 ++++ resources/x509ca/openssl-node1.cnf | 44 ++ resources/x509ca/openssl-node2.cnf | 44 ++ resources/x509ca/openssl-node3.cnf | 44 ++ resources/x509ca/openssl.cnf | 350 ++++++++++++++ resources/x509ca2/openssl-ca.cnf | 86 ++++ resources/x509ca2/openssl-node1.cnf | 44 ++ resources/x509ca2/openssl-node2.cnf | 44 ++ resources/x509ca2/openssl-node3.cnf | 44 ++ .../lib/commons-validator-1.4.0.LICENSE.txt | 202 ++++++++ .../main/org/apache/zookeeper/ClientCnxn.java | 37 +- .../apache/zookeeper/ClientCnxnSocket.java | 5 +- .../apache/zookeeper/ClientCnxnSocketNIO.java | 6 +- .../zookeeper/ClientCnxnSocketNetty.java | 15 +- .../main/org/apache/zookeeper/SSLCertCfg.java | 134 ++++++ .../main/org/apache/zookeeper/ServerCfg.java | 72 +++ .../main/org/apache/zookeeper/ZooKeeper.java | 15 +- .../zookeeper/client/ConnectStringParser.java | 48 +- .../zookeeper/client/FourLetterWordMain.java | 18 +- .../apache/zookeeper/client/HostProvider.java | 11 +- .../zookeeper/client/StaticHostProvider.java | 149 +++--- .../zookeeper/common/X509Exception.java | 4 + .../org/apache/zookeeper/common/X509Util.java | 441 +++++++++++++++--- .../server/NettyServerCnxnFactory.java | 25 +- .../zookeeper/server/ServerCnxnFactory.java | 8 +- .../auth/X509AuthenticationProvider.java | 8 +- .../zookeeper/server/quorum/Follower.java | 7 +- .../zookeeper/server/quorum/Leader.java | 19 +- .../zookeeper/server/quorum/Learner.java | 44 +- .../zookeeper/server/quorum/Observer.java | 8 +- .../server/quorum/QuorumCnxManager.java | 67 ++- .../zookeeper/server/quorum/QuorumPeer.java | 168 ++++++- .../server/quorum/QuorumPeerMain.java | 31 +- .../CertificateVerificationException.java | 36 ++ .../quorum/util/CertificateVerifier.java | 182 ++++++++ .../quorum/util/QuorumSocketFactory.java | 178 +++++++ .../util/ZKDynamicX509TrustManager.java | 178 +++++++ .../quorum/util/ZKPeerX509TrustManager.java | 79 ++++ .../quorum/util/ZKX509TrustManager.java | 101 ++++ .../apache/zookeeper/ClientReconnectTest.java | 5 +- .../zookeeper/CustomHostProviderTest.java | 9 +- .../zookeeper/server/quorum/LearnerTest.java | 9 +- .../zookeeper/server/quorum/Zab1_0Test.java | 17 +- .../test/ConnectStringParserTest.java | 17 +- .../zookeeper/test/LENonTerminateTest.java | 8 +- .../test/StaticHostProviderTest.java | 159 ++++--- 60 files changed, 3304 insertions(+), 372 deletions(-) create mode 100644 conf/multi/README.txt create mode 100644 conf/multi/node1.log4j.properties create mode 100644 conf/multi/node2.log4j.properties create mode 100644 conf/multi/node3.log4j.properties create mode 100755 conf/multi/start_quorum.sh create mode 100644 conf/multi/zoo1.cfg create mode 100644 conf/multi/zoo2.cfg create mode 100644 conf/multi/zoo3.cfg create mode 100644 resources/README.txt create mode 100755 resources/gencert.sh create mode 100755 resources/init.sh create mode 100644 resources/x509ca/openssl-ca.cnf create mode 100644 resources/x509ca/openssl-node1.cnf create mode 100644 resources/x509ca/openssl-node2.cnf create mode 100644 resources/x509ca/openssl-node3.cnf create mode 100644 resources/x509ca/openssl.cnf create mode 100644 resources/x509ca2/openssl-ca.cnf create mode 100644 resources/x509ca2/openssl-node1.cnf create mode 100644 resources/x509ca2/openssl-node2.cnf create mode 100644 resources/x509ca2/openssl-node3.cnf create mode 100644 src/java/lib/commons-validator-1.4.0.LICENSE.txt create mode 100644 src/java/main/org/apache/zookeeper/SSLCertCfg.java create mode 100644 src/java/main/org/apache/zookeeper/ServerCfg.java create mode 100644 src/java/main/org/apache/zookeeper/server/quorum/util/CertificateVerificationException.java create mode 100644 src/java/main/org/apache/zookeeper/server/quorum/util/CertificateVerifier.java create mode 100644 src/java/main/org/apache/zookeeper/server/quorum/util/QuorumSocketFactory.java create mode 100644 src/java/main/org/apache/zookeeper/server/quorum/util/ZKDynamicX509TrustManager.java create mode 100644 src/java/main/org/apache/zookeeper/server/quorum/util/ZKPeerX509TrustManager.java create mode 100644 src/java/main/org/apache/zookeeper/server/quorum/util/ZKX509TrustManager.java diff --git a/.gitignore b/.gitignore index 0cbf40d712d..d08103830e5 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ hs_err_pid* # Eclipse .metadata +*~ .classpath .eclipse/ .idea/ @@ -80,3 +81,20 @@ src/c/depcomp src/c/install-sh src/c/ltmain.sh src/c/missing +resources/x509ca/.rnd +resources/x509ca/java +resources/x509ca/ca +resources/x509ca/certs +resources/x509ca/crl +resources/x509ca/newcerts +resources/x509ca/index.* +resources/x509ca/serial* +resources/x509ca2/.rnd +resources/x509ca2/java +resources/x509ca2/ca +resources/x509ca2/certs +resources/x509ca2/crl +resources/x509ca2/newcerts +resources/x509ca2/index.* +resources/x509ca2/serial* +conf/multi/*.out diff --git a/build.xml b/build.xml index db576f177a5..f953eb00e58 100644 --- a/build.xml +++ b/build.xml @@ -711,6 +711,7 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle.ant"> + diff --git a/conf/multi/README.txt b/conf/multi/README.txt new file mode 100644 index 00000000000..fe29218bae7 --- /dev/null +++ b/conf/multi/README.txt @@ -0,0 +1,4 @@ +Helper to bootstrap a three node configuration. +Needs these three interfaces setup - 127.0.1.1, 127.0.1.2, 127.0.1.3 +Create them as follows: +$ sudo ifconfig lo:1 127.0.1.1 netmask 255.255.255.0 diff --git a/conf/multi/node1.log4j.properties b/conf/multi/node1.log4j.properties new file mode 100644 index 00000000000..b91c8eaf0af --- /dev/null +++ b/conf/multi/node1.log4j.properties @@ -0,0 +1,58 @@ +# Define some default values that can be overridden by system properties +zookeeper.root.logger=INFO, ROLLINGFILE +zookeeper.console.threshold=INFO +zookeeper.log.dir=/tmp/zookeeper/multi/ +zookeeper.log.file=node1.log +zookeeper.log.threshold=DEBUG +zookeeper.tracelog.dir=/tmp/zookeeper/multi/ +zookeeper.tracelog.file=node1_trace.log + +# +# ZooKeeper Logging Configuration +# + +# Format is " (, )+ + +# DEFAULT: console appender only +log4j.rootLogger=${zookeeper.root.logger} + +# Example with rolling log file +#log4j.rootLogger=DEBUG, ROLLINGFILE + +# Example with rolling log file and tracing +#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE + +# +# Log INFO level and above messages to the console +# +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold} +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + +# +# Add ROLLINGFILE to rootLogger to get log file output +# Log DEBUG level and above messages to a log file +log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender +log4j.appender.ROLLINGFILE.Threshold=${zookeeper.log.threshold} +log4j.appender.ROLLINGFILE.File=${zookeeper.log.dir}/${zookeeper.log.file} + +# Max log file size of 10MB +log4j.appender.ROLLINGFILE.MaxFileSize=10MB +# uncomment the next line to limit number of backup files +#log4j.appender.ROLLINGFILE.MaxBackupIndex=10 + +log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout +log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + + +# +# Add TRACEFILE to rootLogger to get log file output +# Log DEBUG level and above messages to a log file +log4j.appender.TRACEFILE=org.apache.log4j.FileAppender +log4j.appender.TRACEFILE.Threshold=TRACE +log4j.appender.TRACEFILE.File=${zookeeper.tracelog.dir}/${zookeeper.tracelog.file} + +log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout +### Notice we are including log4j's NDC here (%x) +log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L][%x] - %m%n diff --git a/conf/multi/node2.log4j.properties b/conf/multi/node2.log4j.properties new file mode 100644 index 00000000000..c496dbb6b09 --- /dev/null +++ b/conf/multi/node2.log4j.properties @@ -0,0 +1,58 @@ +# Define some default values that can be overridden by system properties +zookeeper.root.logger=INFO, ROLLINGFILE +zookeeper.console.threshold=INFO +zookeeper.log.dir=/tmp/zookeeper/multi/ +zookeeper.log.file=node2.log +zookeeper.log.threshold=DEBUG +zookeeper.tracelog.dir=/tmp/zookeeper/multi/ +zookeeper.tracelog.file=node2_trace.log + +# +# ZooKeeper Logging Configuration +# + +# Format is " (, )+ + +# DEFAULT: console appender only +log4j.rootLogger=${zookeeper.root.logger} + +# Example with rolling log file +#log4j.rootLogger=INFO, ROLLINGFILE + +# Example with rolling log file and tracing +#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE + +# +# Log INFO level and above messages to the console +# +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold} +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + +# +# Add ROLLINGFILE to rootLogger to get log file output +# Log DEBUG level and above messages to a log file +log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender +log4j.appender.ROLLINGFILE.Threshold=${zookeeper.log.threshold} +log4j.appender.ROLLINGFILE.File=${zookeeper.log.dir}/${zookeeper.log.file} + +# Max log file size of 10MB +log4j.appender.ROLLINGFILE.MaxFileSize=10MB +# uncomment the next line to limit number of backup files +#log4j.appender.ROLLINGFILE.MaxBackupIndex=10 + +log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout +log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + + +# +# Add TRACEFILE to rootLogger to get log file output +# Log DEBUG level and above messages to a log file +log4j.appender.TRACEFILE=org.apache.log4j.FileAppender +log4j.appender.TRACEFILE.Threshold=TRACE +log4j.appender.TRACEFILE.File=${zookeeper.tracelog.dir}/${zookeeper.tracelog.file} + +log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout +### Notice we are including log4j's NDC here (%x) +log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L][%x] - %m%n diff --git a/conf/multi/node3.log4j.properties b/conf/multi/node3.log4j.properties new file mode 100644 index 00000000000..291234e109e --- /dev/null +++ b/conf/multi/node3.log4j.properties @@ -0,0 +1,58 @@ +# Define some default values that can be overridden by system properties +zookeeper.root.logger=DEBUG, ROLLINGFILE +zookeeper.console.threshold=INFO +zookeeper.log.dir=/tmp/zookeeper/multi/ +zookeeper.log.file=node3.log +zookeeper.log.threshold=DEBUG +zookeeper.tracelog.dir=/tmp/zookeeper/multi/ +zookeeper.tracelog.file=node3_trace.log + +# +# ZooKeeper Logging Configuration +# + +# Format is " (, )+ + +# DEFAULT: console appender only +log4j.rootLogger=${zookeeper.root.logger} + +# Example with rolling log file +#log4j.rootLogger=INFO, ROLLINGFILE + +# Example with rolling log file and tracing +#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE + +# +# Log INFO level and above messages to the console +# +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold} +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + +# +# Add ROLLINGFILE to rootLogger to get log file output +# Log DEBUG level and above messages to a log file +log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender +log4j.appender.ROLLINGFILE.Threshold=${zookeeper.log.threshold} +log4j.appender.ROLLINGFILE.File=${zookeeper.log.dir}/${zookeeper.log.file} + +# Max log file size of 10MB +log4j.appender.ROLLINGFILE.MaxFileSize=10MB +# uncomment the next line to limit number of backup files +#log4j.appender.ROLLINGFILE.MaxBackupIndex=10 + +log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout +log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + + +# +# Add TRACEFILE to rootLogger to get log file output +# Log DEBUG level and above messages to a log file +log4j.appender.TRACEFILE=org.apache.log4j.FileAppender +log4j.appender.TRACEFILE.Threshold=TRACE +log4j.appender.TRACEFILE.File=${zookeeper.tracelog.dir}/${zookeeper.tracelog.file} + +log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout +### Notice we are including log4j's NDC here (%x) +log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L][%x] - %m%n diff --git a/conf/multi/start_quorum.sh b/conf/multi/start_quorum.sh new file mode 100755 index 00000000000..0e21745d117 --- /dev/null +++ b/conf/multi/start_quorum.sh @@ -0,0 +1,45 @@ +#! /bin/bash + +set -x +# Get classpath +cpath=$1 +spath=$2 + +if [ -z "$cpath" ]; then + echo "usage: $0 " + exit 1; +fi + +if [ -z "$spath" ]; then + echo "usage: $0 " + exit 1; +fi + +if [ -z "${cpath}/zookeeper-3.4.8-SNAPSHOT.jar" ]; then + echo "jar not found" + exit 1; +fi + +rm -f ${cpath}/zookeeper.jar > /dev/null 2>&1 +ln -s ${cpath}/zookeeper-3.5.2-alpha-SNAPSHOT.jar ${cpath}/zookeeper.jar + +rm -rf /tmp/zookeeper/* > /dev/null 2>&1 + +mkdir -p /tmp/zookeeper/multi/node1 +echo "01" > /tmp/zookeeper/multi/node1/myid +mkdir -p /tmp/zookeeper/multi/node2 +echo "02" > /tmp/zookeeper/multi/node2/myid +mkdir -p /tmp/zookeeper/multi/node3 +echo "03" > /tmp/zookeeper/multi/node3/myid + +nohup java -Djavax.net.debug=ssl:handshake -Dlog4j.debug -Dlog4j.configuration="file:${PWD}/node1.log4j.properties" -Dzookeeper.admin.enableServer="false" -Dquorum.ssl.enabled="true" -Dzookeeper.ssl.keyStore.location="${spath}/x509ca/java/node1.ks" -Dzookeeper.ssl.keyStore.password="CertPassword1" -Dzookeeper.ssl.trustStore.location="${spath}/x509ca/java/truststore.ks" -Dzookeeper.ssl.trustStore.password="StorePass" -Dzookeeper.ssl.trustStore.rootCA.alias="ca" -cp ${cpath}/zookeeper.jar:${cpath}/lib/*:${cpath}/test/lib/* org.apache.zookeeper.server.quorum.QuorumPeerMain $PWD/zoo1.cfg &>node1.out & + +nohup java -Djavax.net.debug=ssl:handshake -Dlog4j.debug -Dlog4j.configuration="file:${PWD}/node2.log4j.properties" -Dzookeeper.admin.enableServer="false" -Dquorum.ssl.enabled="true" -Dzookeeper.ssl.keyStore.location="${spath}/x509ca/java/node2.ks" -Dzookeeper.ssl.keyStore.password="CertPassword1" -Dzookeeper.ssl.trustStore.location="${spath}/x509ca/java/truststore.ks" -Dzookeeper.ssl.trustStore.password="StorePass" -Dzookeeper.ssl.trustStore.rootCA.alias="ca" -cp ${cpath}/zookeeper.jar:${cpath}/lib/*:${cpath}/test/lib/* org.apache.zookeeper.server.quorum.QuorumPeerMain $PWD/zoo2.cfg &>node2.out & + +nohup java -Djavax.net.debug=ssl:handshake -Dlog4j.debug -Dlog4j.configuration="file:${PWD}/node3.log4j.properties" -Dzookeeper.admin.enableServer="false" -Dquorum.ssl.enabled="true" -Dzookeeper.ssl.keyStore.location="${spath}/x509ca/java/node3.ks" -Dzookeeper.ssl.keyStore.password="CertPassword1" -Dzookeeper.ssl.trustStore.location="${spath}/x509ca/java/truststore.ks" -Dzookeeper.ssl.trustStore.password="StorePass" -Dzookeeper.ssl.trustStore.rootCA.alias="ca" -cp ${cpath}/zookeeper.jar:${cpath}/lib/*:${cpath}/test/lib/* org.apache.zookeeper.server.quorum.QuorumPeerMain $PWD/zoo3.cfg &>node3.out & + + + + + + diff --git a/conf/multi/zoo1.cfg b/conf/multi/zoo1.cfg new file mode 100644 index 00000000000..5568b550155 --- /dev/null +++ b/conf/multi/zoo1.cfg @@ -0,0 +1,5 @@ +initLimit=5 +syncLimit=2 +tickTime=2000 +dataDir=/tmp/zookeeper/multi/node1 +dynamicConfigFile=/home/powell/work/zookeeper/zk-35/zookeeper/conf/multi/zoo1.cfg.dynamic.100000000 diff --git a/conf/multi/zoo2.cfg b/conf/multi/zoo2.cfg new file mode 100644 index 00000000000..d7665a10f40 --- /dev/null +++ b/conf/multi/zoo2.cfg @@ -0,0 +1,5 @@ +initLimit=5 +syncLimit=2 +tickTime=2000 +dataDir=/tmp/zookeeper/multi/node2 +dynamicConfigFile=/home/powell/work/zookeeper/zk-35/zookeeper/conf/multi/zoo2.cfg.dynamic.100000000 diff --git a/conf/multi/zoo3.cfg b/conf/multi/zoo3.cfg new file mode 100644 index 00000000000..f8ed5799ee9 --- /dev/null +++ b/conf/multi/zoo3.cfg @@ -0,0 +1,5 @@ +initLimit=5 +syncLimit=2 +tickTime=2000 +dataDir=/tmp/zookeeper/multi/node3 +dynamicConfigFile=/home/powell/work/zookeeper/zk-35/zookeeper/conf/multi/zoo3.cfg.dynamic.100000000 diff --git a/ivy.xml b/ivy.xml index 4ecb3a60a02..59b1dea9c49 100644 --- a/ivy.xml +++ b/ivy.xml @@ -54,13 +54,15 @@ + + - " + exit 1 +fi +set -x +mdir="." +cdir="${mdir}/ca" +jdir="${mdir}/java" + +# Remove nodeX stuff from java dir +rm -rf ${jdir}/$1* > /dev/null 2>&1 + +# Generate the node certificate +openssl req -config ${mdir}/openssl-$1.cnf -newkey rsa:2048 -sha256 -nodes -out ${jdir}/$1.csr -keyout ${jdir}/$1key.pem -outform PEM + +if [ $? -ne 0 ]; then + echo "error creating keys for ${1}" + exit 1 +fi + +# Verify it. +openssl req -text -noout -verify -in ${jdir}/$1.csr + +if [ $? -ne 0 ]; then + echo "Failed to verify cert for $1" + exit 1 +fi + +# Sign the cert +openssl ca -config ${mdir}/openssl-ca.cnf -policy signing_policy -extensions signing_req -in ${jdir}/$1.csr -out ${jdir}/$1.pem + +if [ $? -ne 0 ]; then + echo "error signing the csr for ${1}" + exit 1 +fi + +# Output pem cert to PCKS12 store along with the CA cert. +openssl pkcs12 -export -out ${jdir}/$1.p12 -inkey ${jdir}/$1key.pem -in ${jdir}/$1.pem -certfile ${mdir}/ca/cacert.pem -name $1 + +if [ $? -ne 0 ]; then + echo "error combining private key and cert along with CA for ${1}" + exit 1 +fi + +# Convert PCKS12 keystore into a JSK keystore +keytool -importkeystore -deststorepass CertPassword1 -destkeypass CertPassword1 -destkeystore ${jdir}/$1.ks -srckeystore ${jdir}/$1.p12 -srcstoretype PKCS12 -srcstorepass CertPassword1 -alias $1 + +if [ $? -ne 0 ]; then + echo "error Creating the keystore for ${1}" + exit 1 +fi diff --git a/resources/init.sh b/resources/init.sh new file mode 100755 index 00000000000..6b5caedf51d --- /dev/null +++ b/resources/init.sh @@ -0,0 +1,39 @@ +#! /bin/bash + +mdir="." +cdir="${mdir}/ca" +jdir="${mdir}/java" + +rm -rf ${mdir}/java/* > /dev/null 2>&1 +rm -rf ${cdir}/* > /dev/null 2>&1 +rm -rf ${mdir}/newcerts/* > /dev/null 2>&1 + +rm ${mdir}/serial* > /dev/null 2>&1 +rm ${mdir}/index* > /dev/null 2>&1 + +echo "01" > ${mdir}/serial +touch ${mdir}/index.txt + +mkdir -p ${mdir}/ca > /dev/null 2>&1 +mkdir -p ${mdir}/certs > /dev/null 2>&1 +mkdir -p ${mdir}/crl > /dev/null 2>&1 +mkdir -p ${mdir}/newcerts > /dev/null 2>&1 +mkdir -p ${jdir} > /dev/null 2>&1 + +# Create CA +openssl req -x509 -config ${mdir}/openssl-ca.cnf -newkey rsa:2048 -sha256 -nodes -out ${cdir}/cacert.pem -keyout ${cdir}/cakey.pem -outform PEM + +if [ $? -ne 0 ]; then + echo "Creation of CA cert failed" + exit 1 +fi + +# Import CA to truststore +keytool -import -file ${cdir}/cacert.pem -alias ca -keystore ${jdir}/truststore.ks -storepass StorePass + +if [ $? -ne 0 ]; then + echo "Import of CA to truststore failed" + exit 1 +fi + +openssl x509 -in ${cdir}/cacert.pem -text -noout diff --git a/resources/x509ca/openssl-ca.cnf b/resources/x509ca/openssl-ca.cnf new file mode 100644 index 00000000000..edb3b550988 --- /dev/null +++ b/resources/x509ca/openssl-ca.cnf @@ -0,0 +1,86 @@ +HOME = . +RANDFILE = $ENV::HOME/.rnd + +#################################################################### +[ ca ] +default_ca = CA_default # The default ca section + +[ CA_default ] + +default_days = 1000 # how long to certify for +default_crl_days= 30 # how long before next CRL +default_md = sha256 # use public key default MD +preserve = no # keep passed DN ordering + +x509_extensions = ca_extensions # The extensions to add to the cert + +email_in_dn = no # Don't concat the email in the DN +copy_extensions = copy # Required to copy SANs from CSR to cert + +base_dir = . +certificate = $base_dir/ca/cacert.pem # The CA certifcate +private_key = $base_dir/ca/cakey.pem # The CA private key +new_certs_dir = $base_dir/newcerts # Location for new certs after signing +database = $base_dir/index.txt # Database index file +serial = $base_dir/serial # The current serial number + +unique_subject = no # Set to 'no' to allow creation of + # several certificates with same subject. +#################################################################### +[ req ] +default_bits = 2048 +default_keyfile = cakey.pem +distinguished_name = ca_distinguished_name +x509_extensions = ca_extensions +string_mask = utf8only + +#################################################################### +[ ca_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = US + +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = California + +localityName = Locality Name (eg, city) +localityName_default = Sunnyvale + +organizationName = Organization Name (eg, company) +organizationName_default = Test CA, Limited + +organizationalUnitName = Organizational Unit (eg, division) +organizationalUnitName_default = Server Research Department + +commonName = Common Name (e.g. server FQDN or YOUR name) +commonName_default = test.root.ca + +emailAddress = Email Address +emailAddress_default = test@ca.org + +#################################################################### +[ ca_extensions ] + +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid:always, issuer +basicConstraints = critical, CA:true +keyUsage = keyCertSign, cRLSign + +#################################################################### +[ signing_policy ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +#################################################################### +[ signing_req ] +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer + +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment + +######################################################################## diff --git a/resources/x509ca/openssl-node1.cnf b/resources/x509ca/openssl-node1.cnf new file mode 100644 index 00000000000..156002afcbb --- /dev/null +++ b/resources/x509ca/openssl-node1.cnf @@ -0,0 +1,44 @@ +HOME = . +RANDFILE = $ENV::HOME/.rnd + +#################################################################### +[ req ] +default_bits = 2048 +default_keyfile = java/node1key.pem +distinguished_name = server_distinguished_name +req_extensions = server_req_extensions +string_mask = utf8only + +#################################################################### +[ server_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = US + +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = California + +localityName = Locality Name (eg, city) +localityName_default = Sunnyvale + +organizationName = Organization Name (eg, company) +organizationName_default = Quorum Inc + +commonName = Common Name (e.g. server FQDN or YOUR name) +commonName_default = node1 + +emailAddress = Email Address +emailAddress_default = node1@quorum.org + +#################################################################### +[ server_req_extensions ] + +subjectKeyIdentifier = hash +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment +subjectAltName = @alternate_names +nsComment = "OpenSSL Generated Certificate" + +#################################################################### +[ alternate_names ] + +DNS.1 = node1.quorum.org diff --git a/resources/x509ca/openssl-node2.cnf b/resources/x509ca/openssl-node2.cnf new file mode 100644 index 00000000000..3570e09631b --- /dev/null +++ b/resources/x509ca/openssl-node2.cnf @@ -0,0 +1,44 @@ +HOME = . +RANDFILE = $ENV::HOME/.rnd + +#################################################################### +[ req ] +default_bits = 2048 +default_keyfile = java/node2key.pem +distinguished_name = server_distinguished_name +req_extensions = server_req_extensions +string_mask = utf8only + +#################################################################### +[ server_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = US + +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = California + +localityName = Locality Name (eg, city) +localityName_default = Sunnyvale + +organizationName = Organization Name (eg, company) +organizationName_default = Quorum Inc + +commonName = Common Name (e.g. server FQDN or YOUR name) +commonName_default = node2 + +emailAddress = Email Address +emailAddress_default = node2@quorum.org + +#################################################################### +[ server_req_extensions ] + +subjectKeyIdentifier = hash +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment +subjectAltName = @alternate_names +nsComment = "OpenSSL Generated Certificate" + +#################################################################### +[ alternate_names ] + +DNS.1 = node2.quorum.org diff --git a/resources/x509ca/openssl-node3.cnf b/resources/x509ca/openssl-node3.cnf new file mode 100644 index 00000000000..9082efbaac5 --- /dev/null +++ b/resources/x509ca/openssl-node3.cnf @@ -0,0 +1,44 @@ +HOME = . +RANDFILE = $ENV::HOME/.rnd + +#################################################################### +[ req ] +default_bits = 2048 +default_keyfile = java/node3key.pem +distinguished_name = server_distinguished_name +req_extensions = server_req_extensions +string_mask = utf8only + +#################################################################### +[ server_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = US + +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = California + +localityName = Locality Name (eg, city) +localityName_default = Sunnyvale + +organizationName = Organization Name (eg, company) +organizationName_default = Quorum Inc + +commonName = Common Name (e.g. server FQDN or YOUR name) +commonName_default = node3 + +emailAddress = Email Address +emailAddress_default = node3@quorum.org + +#################################################################### +[ server_req_extensions ] + +subjectKeyIdentifier = hash +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment +subjectAltName = @alternate_names +nsComment = "OpenSSL Generated Certificate" + +#################################################################### +[ alternate_names ] + +DNS.1 = node3.quorum.org diff --git a/resources/x509ca/openssl.cnf b/resources/x509ca/openssl.cnf new file mode 100644 index 00000000000..e1bab492076 --- /dev/null +++ b/resources/x509ca/openssl.cnf @@ -0,0 +1,350 @@ +# +# OpenSSL example configuration file. +# This is mostly being used for generation of certificate requests. +# + +# This definition stops the following lines choking if HOME isn't +# defined. +HOME = . +RANDFILE = $ENV::HOME/.rnd + +# Extra OBJECT IDENTIFIER info: +#oid_file = $ENV::HOME/.oid +oid_section = new_oids + +# To use this configuration file with the "-extfile" option of the +# "openssl x509" utility, name here the section containing the +# X.509v3 extensions to use: +# extensions = +# (Alternatively, use a configuration file that has only +# X.509v3 extensions in its main [= default] section.) + +[ new_oids ] + +# We can add new OIDs in here for use by 'ca', 'req' and 'ts'. +# Add a simple OID like this: +# testoid1=1.2.3.4 +# Or use config file substitution like this: +# testoid2=${testoid1}.5.6 + +# Policies used by the TSA examples. +tsa_policy1 = 1.2.3.4.1 +tsa_policy2 = 1.2.3.4.5.6 +tsa_policy3 = 1.2.3.4.5.7 + +#################################################################### +[ ca ] +default_ca = CA_default # The default ca section + +#################################################################### +[ CA_default ] + +dir = x509ca # Where everything is kept +certs = $dir/certs # Where the issued certs are kept +crl_dir = $dir/crl # Where the issued crl are kept +database = $dir/index.txt # database index file. +#unique_subject = no # Set to 'no' to allow creation of + # several ctificates with same subject. +new_certs_dir = $dir/newcerts # default place for new certs. + +certificate = $dir/ca/new_ca.pem # The CA certificate +serial = $dir/serial # The current serial number +crlnumber = $dir/crlnumber # the current crl number + # must be commented out to leave a V1 CRL +crl = $dir/crl.pem # The current CRL +private_key = $dir/ca/new_ca_pk.pem # The private key +RANDFILE = $dir/ca/.rand # private random number file + +x509_extensions = usr_cert # The extentions to add to the cert + +# Comment out the following two lines for the "traditional" +# (and highly broken) format. +name_opt = ca_default # Subject Name options +cert_opt = ca_default # Certificate field options + +# Extension copying option: use with caution. +# copy_extensions = copy + +# Extensions to add to a CRL. Note: Netscape communicator chokes on V2 CRLs +# so this is commented out by default to leave a V1 CRL. +# crlnumber must also be commented out to leave a V1 CRL. +# crl_extensions = crl_ext + +default_days = 365 # how long to certify for +default_crl_days= 30 # how long before next CRL +default_md = sha256 # use public key default MD +preserve = no # keep passed DN ordering + +# A few difference way of specifying how similar the request should look +# For type CA, the listed attributes must be the same, and the optional +# and supplied fields are just that :-) +policy = policy_match + +# For the CA policy +[ policy_match ] +countryName = match +stateOrProvinceName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +# For the 'anything' policy +# At this point in time, you must list all acceptable 'object' +# types. +[ policy_anything ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +#################################################################### +[ req ] +default_bits = 2048 +default_keyfile = privkey.pem +distinguished_name = req_distinguished_name +attributes = req_attributes +x509_extensions = v3_ca # The extentions to add to the self signed cert + +# Passwords for private keys if not present they will be prompted for +# input_password = secret +# output_password = secret + +# This sets a mask for permitted string types. There are several options. +# default: PrintableString, T61String, BMPString. +# pkix : PrintableString, BMPString (PKIX recommendation before 2004) +# utf8only: only UTF8Strings (PKIX recommendation after 2004). +# nombstr : PrintableString, T61String (no BMPStrings or UTF8Strings). +# MASK:XXXX a literal mask value. +# WARNING: ancient versions of Netscape crash on BMPStrings or UTF8Strings. +string_mask = utf8only + +# req_extensions = v3_req # The extensions to add to a certificate request + +[ req_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = AU +countryName_min = 2 +countryName_max = 2 + +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = Some-State + +localityName = Locality Name (eg, city) + +0.organizationName = Organization Name (eg, company) +0.organizationName_default = Internet Widgits Pty Ltd + +# we can do this but it is not needed normally :-) +#1.organizationName = Second Organization Name (eg, company) +#1.organizationName_default = World Wide Web Pty Ltd + +organizationalUnitName = Organizational Unit Name (eg, section) +#organizationalUnitName_default = + +commonName = Common Name (e.g. server FQDN or YOUR name) +commonName_max = 64 + +emailAddress = Email Address +emailAddress_max = 64 + +# SET-ex3 = SET extension number 3 + +[ req_attributes ] +challengePassword = A challenge password +challengePassword_min = 4 +challengePassword_max = 20 + +unstructuredName = An optional company name + +[ usr_cert ] + +# These extensions are added when 'ca' signs a request. + +# This goes against PKIX guidelines but some CAs do it and some software +# requires this to avoid interpreting an end user certificate as a CA. + +basicConstraints=CA:FALSE + +# Here are some examples of the usage of nsCertType. If it is omitted +# the certificate can be used for anything *except* object signing. + +# This is OK for an SSL server. +# nsCertType = server + +# For an object signing certificate this would be used. +# nsCertType = objsign + +# For normal client use this is typical +# nsCertType = client, email + +# and for everything including object signing: +# nsCertType = client, email, objsign + +# This is typical in keyUsage for a client certificate. +# keyUsage = nonRepudiation, digitalSignature, keyEncipherment + +# This will be displayed in Netscape's comment listbox. +nsComment = "OpenSSL Generated Certificate" + +# PKIX recommendations harmless if included in all certificates. +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer + +# This stuff is for subjectAltName and issuerAltname. +# Import the email address. +# subjectAltName=email:copy +# An alternative to produce certificates that aren't +# deprecated according to PKIX. +# subjectAltName=email:move + +# Copy subject details +# issuerAltName=issuer:copy + +#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem +#nsBaseUrl +#nsRevocationUrl +#nsRenewalUrl +#nsCaPolicyUrl +#nsSslServerName + +# This is required for TSA certificates. +# extendedKeyUsage = critical,timeStamping + +[ v3_req ] + +# Extensions to add to a certificate request + +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment + +[ v3_ca ] + + +# Extensions for a typical CA + + +# PKIX recommendation. + +subjectKeyIdentifier=hash + +authorityKeyIdentifier=keyid:always,issuer + +# This is what PKIX recommends but some broken software chokes on critical +# extensions. +#basicConstraints = critical,CA:true +# So we do this instead. +basicConstraints = CA:true + +# Key usage: this is typical for a CA certificate. However since it will +# prevent it being used as an test self-signed certificate it is best +# left out by default. +# keyUsage = cRLSign, keyCertSign + +# Some might want this also +# nsCertType = sslCA, emailCA + +# Include email address in subject alt name: another PKIX recommendation +# subjectAltName=email:copy +# Copy issuer details +# issuerAltName=issuer:copy + +# DER hex encoding of an extension: beware experts only! +# obj=DER:02:03 +# Where 'obj' is a standard or added object +# You can even override a supported extension: +# basicConstraints= critical, DER:30:03:01:01:FF + +[ crl_ext ] + +# CRL extensions. +# Only issuerAltName and authorityKeyIdentifier make any sense in a CRL. + +# issuerAltName=issuer:copy +authorityKeyIdentifier=keyid:always + +[ proxy_cert_ext ] +# These extensions should be added when creating a proxy certificate + +# This goes against PKIX guidelines but some CAs do it and some software +# requires this to avoid interpreting an end user certificate as a CA. + +basicConstraints=CA:FALSE + +# Here are some examples of the usage of nsCertType. If it is omitted +# the certificate can be used for anything *except* object signing. + +# This is OK for an SSL server. +# nsCertType = server + +# For an object signing certificate this would be used. +# nsCertType = objsign + +# For normal client use this is typical +# nsCertType = client, email + +# and for everything including object signing: +# nsCertType = client, email, objsign + +# This is typical in keyUsage for a client certificate. +# keyUsage = nonRepudiation, digitalSignature, keyEncipherment + +# This will be displayed in Netscape's comment listbox. +nsComment = "OpenSSL Generated Certificate" + +# PKIX recommendations harmless if included in all certificates. +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer + +# This stuff is for subjectAltName and issuerAltname. +# Import the email address. +# subjectAltName=email:copy +# An alternative to produce certificates that aren't +# deprecated according to PKIX. +# subjectAltName=email:move + +# Copy subject details +# issuerAltName=issuer:copy + +#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem +#nsBaseUrl +#nsRevocationUrl +#nsRenewalUrl +#nsCaPolicyUrl +#nsSslServerName + +# This really needs to be in place for it to be a proxy certificate. +proxyCertInfo=critical,language:id-ppl-anyLanguage,pathlen:3,policy:foo + +#################################################################### +[ tsa ] + +default_tsa = tsa_config1 # the default TSA section + +[ tsa_config1 ] + +# These are used by the TSA reply generation only. +dir = ./demoCA # TSA root directory +serial = $dir/tsaserial # The current serial number (mandatory) +crypto_device = builtin # OpenSSL engine to use for signing +signer_cert = $dir/tsacert.pem # The TSA signing certificate + # (optional) +certs = $dir/cacert.pem # Certificate chain to include in reply + # (optional) +signer_key = $dir/private/tsakey.pem # The TSA private key (optional) + +default_policy = tsa_policy1 # Policy if request did not specify it + # (optional) +other_policies = tsa_policy2, tsa_policy3 # acceptable policies (optional) +digests = md5, sha1 # Acceptable message digests (mandatory) +accuracy = secs:1, millisecs:500, microsecs:100 # (optional) +clock_precision_digits = 0 # number of digits after dot. (optional) +ordering = yes # Is ordering defined for timestamps? + # (optional, default: no) +tsa_name = yes # Must the TSA name be included in the reply? + # (optional, default: no) +ess_cert_id_chain = no # Must the ESS cert id chain be included? + # (optional, default: no) diff --git a/resources/x509ca2/openssl-ca.cnf b/resources/x509ca2/openssl-ca.cnf new file mode 100644 index 00000000000..eb21a63de9d --- /dev/null +++ b/resources/x509ca2/openssl-ca.cnf @@ -0,0 +1,86 @@ +HOME = . +RANDFILE = $ENV::HOME/.rnd + +#################################################################### +[ ca ] +default_ca = CA_default # The default ca section + +[ CA_default ] + +default_days = 1000 # how long to certify for +default_crl_days= 30 # how long before next CRL +default_md = sha256 # use public key default MD +preserve = no # keep passed DN ordering + +x509_extensions = ca_extensions # The extensions to add to the cert + +email_in_dn = no # Don't concat the email in the DN +copy_extensions = copy # Required to copy SANs from CSR to cert + +base_dir = . +certificate = $base_dir/ca/cacert.pem # The CA certifcate +private_key = $base_dir/ca/cakey.pem # The CA private key +new_certs_dir = $base_dir/newcerts # Location for new certs after signing +database = $base_dir/index.txt # Database index file +serial = $base_dir/serial # The current serial number + +unique_subject = no # Set to 'no' to allow creation of + # several certificates with same subject. +#################################################################### +[ req ] +default_bits = 4096 +default_keyfile = cakey.pem +distinguished_name = ca_distinguished_name +x509_extensions = ca_extensions +string_mask = utf8only + +#################################################################### +[ ca_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = US + +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = California + +localityName = Locality Name (eg, city) +localityName_default = Sunnyvale + +organizationName = Organization Name (eg, company) +organizationName_default = Test CA, Limited + +organizationalUnitName = Organizational Unit (eg, division) +organizationalUnitName_default = Server Research Department + +commonName = Common Name (e.g. server FQDN or YOUR name) +commonName_default = test.root.ca + +emailAddress = Email Address +emailAddress_default = test@ca.org + +#################################################################### +[ ca_extensions ] + +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid:always, issuer +basicConstraints = critical, CA:true +keyUsage = keyCertSign, cRLSign + +#################################################################### +[ signing_policy ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +#################################################################### +[ signing_req ] +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer + +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment + +######################################################################## diff --git a/resources/x509ca2/openssl-node1.cnf b/resources/x509ca2/openssl-node1.cnf new file mode 100644 index 00000000000..156002afcbb --- /dev/null +++ b/resources/x509ca2/openssl-node1.cnf @@ -0,0 +1,44 @@ +HOME = . +RANDFILE = $ENV::HOME/.rnd + +#################################################################### +[ req ] +default_bits = 2048 +default_keyfile = java/node1key.pem +distinguished_name = server_distinguished_name +req_extensions = server_req_extensions +string_mask = utf8only + +#################################################################### +[ server_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = US + +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = California + +localityName = Locality Name (eg, city) +localityName_default = Sunnyvale + +organizationName = Organization Name (eg, company) +organizationName_default = Quorum Inc + +commonName = Common Name (e.g. server FQDN or YOUR name) +commonName_default = node1 + +emailAddress = Email Address +emailAddress_default = node1@quorum.org + +#################################################################### +[ server_req_extensions ] + +subjectKeyIdentifier = hash +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment +subjectAltName = @alternate_names +nsComment = "OpenSSL Generated Certificate" + +#################################################################### +[ alternate_names ] + +DNS.1 = node1.quorum.org diff --git a/resources/x509ca2/openssl-node2.cnf b/resources/x509ca2/openssl-node2.cnf new file mode 100644 index 00000000000..3570e09631b --- /dev/null +++ b/resources/x509ca2/openssl-node2.cnf @@ -0,0 +1,44 @@ +HOME = . +RANDFILE = $ENV::HOME/.rnd + +#################################################################### +[ req ] +default_bits = 2048 +default_keyfile = java/node2key.pem +distinguished_name = server_distinguished_name +req_extensions = server_req_extensions +string_mask = utf8only + +#################################################################### +[ server_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = US + +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = California + +localityName = Locality Name (eg, city) +localityName_default = Sunnyvale + +organizationName = Organization Name (eg, company) +organizationName_default = Quorum Inc + +commonName = Common Name (e.g. server FQDN or YOUR name) +commonName_default = node2 + +emailAddress = Email Address +emailAddress_default = node2@quorum.org + +#################################################################### +[ server_req_extensions ] + +subjectKeyIdentifier = hash +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment +subjectAltName = @alternate_names +nsComment = "OpenSSL Generated Certificate" + +#################################################################### +[ alternate_names ] + +DNS.1 = node2.quorum.org diff --git a/resources/x509ca2/openssl-node3.cnf b/resources/x509ca2/openssl-node3.cnf new file mode 100644 index 00000000000..9082efbaac5 --- /dev/null +++ b/resources/x509ca2/openssl-node3.cnf @@ -0,0 +1,44 @@ +HOME = . +RANDFILE = $ENV::HOME/.rnd + +#################################################################### +[ req ] +default_bits = 2048 +default_keyfile = java/node3key.pem +distinguished_name = server_distinguished_name +req_extensions = server_req_extensions +string_mask = utf8only + +#################################################################### +[ server_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = US + +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = California + +localityName = Locality Name (eg, city) +localityName_default = Sunnyvale + +organizationName = Organization Name (eg, company) +organizationName_default = Quorum Inc + +commonName = Common Name (e.g. server FQDN or YOUR name) +commonName_default = node3 + +emailAddress = Email Address +emailAddress_default = node3@quorum.org + +#################################################################### +[ server_req_extensions ] + +subjectKeyIdentifier = hash +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment +subjectAltName = @alternate_names +nsComment = "OpenSSL Generated Certificate" + +#################################################################### +[ alternate_names ] + +DNS.1 = node3.quorum.org diff --git a/src/java/lib/commons-validator-1.4.0.LICENSE.txt b/src/java/lib/commons-validator-1.4.0.LICENSE.txt new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/src/java/lib/commons-validator-1.4.0.LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/src/java/main/org/apache/zookeeper/ClientCnxn.java b/src/java/main/org/apache/zookeeper/ClientCnxn.java index 9e6c154eca5..0829d30989e 100644 --- a/src/java/main/org/apache/zookeeper/ClientCnxn.java +++ b/src/java/main/org/apache/zookeeper/ClientCnxn.java @@ -787,7 +787,7 @@ public RWServerFoundException(String msg) { super(msg); } } - + /** * This class services the outgoing request queue and generates the heart * beats. It also spawns the ReadThread. @@ -1041,7 +1041,7 @@ private void sendPing() { queuePacket(h, null, null, null, null, null, null, null, null); } - private InetSocketAddress rwServerAddress = null; + private ServerCfg rwServerCfg = null; private final static int minPingRwTimeout = 100; @@ -1063,15 +1063,16 @@ private void startConnect() throws IOException { } state = States.CONNECTING; - InetSocketAddress addr; - if (rwServerAddress != null) { - addr = rwServerAddress; - rwServerAddress = null; + ServerCfg serverCfg; + if (rwServerCfg != null) { + serverCfg = rwServerCfg; + rwServerCfg = null; } else { - addr = hostProvider.next(1000); + serverCfg = hostProvider.next(1000); } - String hostPort = addr.getHostString() + ":" + addr.getPort(); + String hostPort = serverCfg.getInetAddress().getHostString() + ":" + + serverCfg.getInetAddress().getPort(); MDC.put("myid", hostPort); setName(getName().replaceAll("\\(.*\\)", "(" + hostPort + ")")); if (clientConfig.isSaslClientEnabled()) { @@ -1079,7 +1080,7 @@ private void startConnect() throws IOException { if (zooKeeperSaslClient != null) { zooKeeperSaslClient.shutdown(); } - zooKeeperSaslClient = new ZooKeeperSaslClient(getServerPrincipal(addr), clientConfig); + zooKeeperSaslClient = new ZooKeeperSaslClient(getServerPrincipal(serverCfg.getInetAddress()), clientConfig); } catch (LoginException e) { // An authentication error occurred when the SASL client tried to initialize: // for Kerberos this means that the client failed to authenticate with the KDC. @@ -1093,9 +1094,9 @@ private void startConnect() throws IOException { saslLoginFailed = true; } } - logStartConnect(addr); + logStartConnect(serverCfg.getInetAddress()); - clientCnxnSocket.connect(addr); + clientCnxnSocket.connect(serverCfg); } private String getServerPrincipal(InetSocketAddress addr) { @@ -1271,14 +1272,15 @@ public void run() { private void pingRwServer() throws RWServerFoundException { String result = null; - InetSocketAddress addr = hostProvider.next(0); - LOG.info("Checking server " + addr + " for being r/w." + - " Timeout " + pingRwTimeout); + final ServerCfg serverCfg = hostProvider.next(0); + LOG.info("Checking server " + serverCfg.getInetAddress() + + " for being r/w." + " Timeout " + pingRwTimeout); Socket sock = null; BufferedReader br = null; try { - sock = new Socket(addr.getHostString(), addr.getPort()); + sock = new Socket(serverCfg.getInetAddress().getHostString(), + serverCfg.getInetAddress().getPort()); sock.setSoLinger(false, -1); sock.setSoTimeout(1000); sock.setTcpNoDelay(true); @@ -1315,9 +1317,10 @@ private void pingRwServer() throws RWServerFoundException { pingRwTimeout = minPingRwTimeout; // save the found address so that it's used during the next // connection attempt - rwServerAddress = addr; + rwServerCfg = serverCfg; throw new RWServerFoundException("Majority server found at " - + addr.getHostString() + ":" + addr.getPort()); + + serverCfg.getInetAddress().getHostString() + ":" + + serverCfg.getInetAddress().getPort()); } } diff --git a/src/java/main/org/apache/zookeeper/ClientCnxnSocket.java b/src/java/main/org/apache/zookeeper/ClientCnxnSocket.java index 0e5316d4eed..b78b0cded8e 100644 --- a/src/java/main/org/apache/zookeeper/ClientCnxnSocket.java +++ b/src/java/main/org/apache/zookeeper/ClientCnxnSocket.java @@ -19,7 +19,6 @@ package org.apache.zookeeper; import java.io.IOException; -import java.net.InetSocketAddress; import java.net.SocketAddress; import java.nio.ByteBuffer; import java.text.MessageFormat; @@ -29,8 +28,8 @@ import org.apache.jute.BinaryInputArchive; import org.apache.zookeeper.ClientCnxn.Packet; import org.apache.zookeeper.client.ZKClientConfig; -import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.common.Time; +import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.proto.ConnectResponse; import org.apache.zookeeper.server.ByteBufferInputStream; import org.slf4j.Logger; @@ -155,7 +154,7 @@ void readConnectResult() throws IOException { abstract boolean isConnected(); - abstract void connect(InetSocketAddress addr) throws IOException; + abstract void connect(final ServerCfg serverCfg) throws IOException; /** * Returns the address to which the socket is connected. diff --git a/src/java/main/org/apache/zookeeper/ClientCnxnSocketNIO.java b/src/java/main/org/apache/zookeeper/ClientCnxnSocketNIO.java index f17a8197150..20bc16534f1 100644 --- a/src/java/main/org/apache/zookeeper/ClientCnxnSocketNIO.java +++ b/src/java/main/org/apache/zookeeper/ClientCnxnSocketNIO.java @@ -279,12 +279,12 @@ void registerAndConnect(SocketChannel sock, InetSocketAddress addr) } @Override - void connect(InetSocketAddress addr) throws IOException { + void connect(final ServerCfg serverCfg) throws IOException { SocketChannel sock = createSock(); try { - registerAndConnect(sock, addr); + registerAndConnect(sock, serverCfg.getInetAddress()); } catch (IOException e) { - LOG.error("Unable to open socket to " + addr); + LOG.error("Unable to open socket to " + serverCfg.getInetAddress()); sock.close(); throw e; } diff --git a/src/java/main/org/apache/zookeeper/ClientCnxnSocketNetty.java b/src/java/main/org/apache/zookeeper/ClientCnxnSocketNetty.java index 97af9da698b..4a274adef5d 100644 --- a/src/java/main/org/apache/zookeeper/ClientCnxnSocketNetty.java +++ b/src/java/main/org/apache/zookeeper/ClientCnxnSocketNetty.java @@ -46,7 +46,6 @@ import javax.net.ssl.SSLEngine; import java.io.IOException; -import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Iterator; import java.util.List; @@ -107,16 +106,16 @@ boolean isConnected() { } @Override - void connect(InetSocketAddress addr) throws IOException { + void connect(final ServerCfg serverCfg) throws IOException { firstConnect = new CountDownLatch(1); ClientBootstrap bootstrap = new ClientBootstrap(channelFactory); - bootstrap.setPipelineFactory(new ZKClientPipelineFactory()); + bootstrap.setPipelineFactory(new ZKClientPipelineFactory(serverCfg)); bootstrap.setOption("soLinger", -1); bootstrap.setOption("tcpNoDelay", true); - connectFuture = bootstrap.connect(addr); + connectFuture = bootstrap.connect(serverCfg.getInetAddress()); connectFuture.addListener(new ChannelFutureListener() { @Override public void operationComplete(ChannelFuture channelFuture) throws Exception { @@ -345,9 +344,14 @@ public static Packet getInstance() { * connection implementation. */ private class ZKClientPipelineFactory implements ChannelPipelineFactory { + private final ServerCfg serverCfg; private SSLContext sslContext = null; private SSLEngine sslEngine = null; + public ZKClientPipelineFactory(final ServerCfg serverCfg) { + this.serverCfg = serverCfg; + } + @Override public ChannelPipeline getPipeline() throws Exception { ChannelPipeline pipeline = Channels.pipeline(); @@ -362,7 +366,8 @@ public ChannelPipeline getPipeline() throws Exception { // Basically we only need to create it once. private synchronized void initSSL(ChannelPipeline pipeline) throws SSLContextException { if (sslContext == null || sslEngine == null) { - sslContext = X509Util.createSSLContext(clientConfig); + sslContext = X509Util.createSSLContext(clientConfig, + serverCfg.getInetAddress(), serverCfg.getSslCertCfg()); sslEngine = sslContext.createSSLEngine(); sslEngine.setUseClientMode(true); } diff --git a/src/java/main/org/apache/zookeeper/SSLCertCfg.java b/src/java/main/org/apache/zookeeper/SSLCertCfg.java new file mode 100644 index 00000000000..7ac74a62f05 --- /dev/null +++ b/src/java/main/org/apache/zookeeper/SSLCertCfg.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper; + +import java.security.MessageDigest; +import java.util.HashMap; +import java.util.Map; + +import org.apache.zookeeper.common.X509Util; +import org.apache.zookeeper.server.quorum.QuorumPeerConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static javax.xml.bind.DatatypeConverter.printHexBinary; + +public class SSLCertCfg { + private static final Logger LOG = LoggerFactory.getLogger(SSLCertCfg.class); + private final CertType certType; + private final String certFingerPrintStr; + public enum CertType { + NONE, SELF, CA; + } + + public SSLCertCfg() { + certType = CertType.NONE; + certFingerPrintStr = null; + } + + public SSLCertCfg(final CertType certType, + final String certFingerPrintStr) { + this.certType = certType; + this.certFingerPrintStr = certFingerPrintStr; + } + + public boolean isSelfSigned() { + return certType == CertType.SELF; + } + + public boolean isCASigned() { + return certType == CertType.CA; + } + + public String getCertFingerPrintStr() { + return certFingerPrintStr; + } + + public static SSLCertCfg parseCertCfgStr(final String certCfgStr) + throws QuorumPeerConfig.ConfigException { + SSLCertCfg.CertType certType = SSLCertCfg.CertType.NONE; + int fpIndex = Integer.MAX_VALUE; + final String[] parts = certCfgStr.split(":"); + final Map propKvMap = + getKeyAndIndexMap(certCfgStr); + if (propKvMap.containsKey("cert") && + propKvMap.containsKey("cacert")) { + final String errStr = "Server string has both self signed " + + "cert and ca cert: " + certCfgStr; + throw new QuorumPeerConfig.ConfigException(errStr); + } else if (propKvMap.containsKey("cert")) { + certType = SSLCertCfg.CertType.SELF; + fpIndex = propKvMap.get("cert") + 1; + if (parts.length < fpIndex) { + final String errStr = "No fingerprint provided for self " + + "signed, server cfg string: " + certCfgStr; + throw new QuorumPeerConfig.ConfigException(errStr); + } + } else if (propKvMap.containsKey("cacert")) { + certType = SSLCertCfg.CertType.SELF; + fpIndex = propKvMap.get("cacert") + 1; + } + + if (fpIndex != Integer.MAX_VALUE && + parts.length > fpIndex) { + LOG.debug("certCfgStr: " + certCfgStr + ", cert type:" + certType + + ", fp:" + parts[fpIndex]); + if (getMessageDigest(parts[fpIndex]) != null) { + return new SSLCertCfg(certType, parts[fpIndex]); + } + } + + return new SSLCertCfg(); + } + + private static Map getKeyAndIndexMap( + final String cfgStr) { + final Map propKvMap = new HashMap<>(); + final String[] parts = cfgStr.split(":"); + for (int i = 0; i < parts.length; i++) { + propKvMap.put(parts[i].trim().toLowerCase(), i); + } + + return propKvMap; + } + + /** + * Given a fingerprint get the supported message digest object for that. + * @param fp fingerprint. + * @return MessageDigest, null on error + */ + private static MessageDigest getMessageDigest(final String fp) + throws QuorumPeerConfig.ConfigException { + // Check for supported algos for the given fingerprint if cannot + // validate throw exception. + MessageDigest md = X509Util.getSupportedMessageDigestForFpStr(fp); + if (md == null) { + final String errStr = "Algo in fingerprint: " + fp + + " not supported, bailing out"; + throw new QuorumPeerConfig.ConfigException(errStr); + } + + return md; + } + + public static String getDigestToCertFp(final MessageDigest md) { + return md.getAlgorithm().toLowerCase() + "-" + + printHexBinary(md.digest()).toLowerCase(); + } +} diff --git a/src/java/main/org/apache/zookeeper/ServerCfg.java b/src/java/main/org/apache/zookeeper/ServerCfg.java new file mode 100644 index 00000000000..65f3b17e9d0 --- /dev/null +++ b/src/java/main/org/apache/zookeeper/ServerCfg.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper; + +import java.net.InetSocketAddress; + +public class ServerCfg { + private final String hostStr; + private final InetSocketAddress address; + private final SSLCertCfg sslCertCfg; + + public ServerCfg(final String hostStr, + final InetSocketAddress address, + final SSLCertCfg sslCertCfg) { + this.hostStr = hostStr; + this.address = address; + this.sslCertCfg = sslCertCfg; + } + + public ServerCfg(final String hostStr, final InetSocketAddress address) { + this.hostStr = hostStr; + this.address = address; + this.sslCertCfg = new SSLCertCfg(); + } + + public String getHostStr() { + return hostStr; + } + + public InetSocketAddress getInetAddress() { + return address; + } + + public SSLCertCfg getSslCertCfg() { + return sslCertCfg; + } + + @Override + public boolean equals(final Object other) { + return other instanceof ServerCfg && + this.hostStr.equals(((ServerCfg)other).getHostStr()); + } + + @Override + public int hashCode() { + return hostStr.hashCode(); + } + + public String getHostString() { + return this.address.getHostString(); + } + + public int getPort() { + return this.address.getPort(); + } +} diff --git a/src/java/main/org/apache/zookeeper/ZooKeeper.java b/src/java/main/org/apache/zookeeper/ZooKeeper.java index b8c8008e8de..e5d0fcd71f2 100644 --- a/src/java/main/org/apache/zookeeper/ZooKeeper.java +++ b/src/java/main/org/apache/zookeeper/ZooKeeper.java @@ -154,6 +154,10 @@ public class ZooKeeper implements AutoCloseable { @Deprecated public static final String SECURE_CLIENT = "zookeeper.client.secure"; + // TODO: XXX: Move to ZKConfig + public static final String PEER_HOST_CERT_FINGERPRINT + = "zookeeper.client.peer.cert.fingerprint"; + protected final ClientCnxn cnxn; private static final Logger LOG; static { @@ -204,13 +208,13 @@ public class ZooKeeper implements AutoCloseable { * @throws IOException in cases of network failure */ public void updateServerList(String connectString) throws IOException { - ConnectStringParser connectStringParser = new ConnectStringParser(connectString); - Collection serverAddresses = connectStringParser.getServerAddresses(); + final ConnectStringParser connectStringParser = new ConnectStringParser(connectString); + final Collection serversCfg = connectStringParser.getServersCfg(); ClientCnxnSocket clientCnxnSocket = cnxn.sendThread.getClientCnxnSocket(); InetSocketAddress currentHost = (InetSocketAddress) clientCnxnSocket.getRemoteSocketAddress(); - boolean reconfigMode = hostProvider.updateServerList(serverAddresses, currentHost); + boolean reconfigMode = hostProvider.updateServerList(serversCfg, currentHost); // cause disconnection - this will cause next to be called // which will in turn call nextReconfigMode @@ -1186,6 +1190,9 @@ public ZooKeeper(String connectString, int sessionTimeout, Watcher watcher, * would be relative to this root - ie getting/setting/etc... * "/foo/bar" would result in operations being run on * "/app/a/foo/bar" (from the server perspective). + * With SSL support the string might look like this: + * "127.0.0.1:3000:SHA-256-XXXXX,127.0.0.1:3001:SHA-256-XXXXX, + * 127.0.0.1:3002:SHA-256-XXXXX" * @param sessionTimeout * session timeout in milliseconds * @param watcher @@ -1216,7 +1223,7 @@ public ZooKeeper(String connectString, int sessionTimeout, Watcher watcher, // default hostprovider private static HostProvider createDefaultHostProvider(String connectString) { return new StaticHostProvider( - new ConnectStringParser(connectString).getServerAddresses()); + new ConnectStringParser(connectString).getServersCfg()); } // VisibleForTesting diff --git a/src/java/main/org/apache/zookeeper/client/ConnectStringParser.java b/src/java/main/org/apache/zookeeper/client/ConnectStringParser.java index 085e44dfb51..1a23c60f1fe 100644 --- a/src/java/main/org/apache/zookeeper/client/ConnectStringParser.java +++ b/src/java/main/org/apache/zookeeper/client/ConnectStringParser.java @@ -18,12 +18,17 @@ package org.apache.zookeeper.client; -import org.apache.zookeeper.common.PathUtils; - import java.net.InetSocketAddress; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.List; +import org.apache.zookeeper.SSLCertCfg; +import org.apache.zookeeper.ServerCfg; +import org.apache.zookeeper.common.PathUtils; +import org.apache.zookeeper.server.quorum.QuorumPeerConfig; + import static org.apache.zookeeper.common.StringUtils.split; /** @@ -41,7 +46,7 @@ public final class ConnectStringParser { private final String chrootPath; - private final ArrayList serverAddresses = new ArrayList(); + private final ArrayList serverCfgList = new ArrayList<>(); /** * @@ -65,18 +70,37 @@ public ConnectStringParser(String connectString) { this.chrootPath = null; } + List hostsList = split(connectString,","); for (String host : hostsList) { + final String[] hostStrParts = host.split(":"); int port = DEFAULT_PORT; - int pidx = host.lastIndexOf(':'); - if (pidx >= 0) { - // otherwise : is at the end of the string, ignore - if (pidx < host.length() - 1) { - port = Integer.parseInt(host.substring(pidx + 1)); + boolean noPort = false; + if (hostStrParts.length > 1) { + try { + port = Integer.parseInt(hostStrParts[1]); + } catch (NumberFormatException exp) { + // ok nothing to do here!. + noPort = true; + } + } + + try { + if (hostStrParts.length > 2 || noPort) { + serverCfgList.add( + new ServerCfg(hostStrParts[0], + InetSocketAddress.createUnresolved(host, + port), + SSLCertCfg.parseCertCfgStr(host))); + } else { + serverCfgList.add( + new ServerCfg(hostStrParts[0], + InetSocketAddress.createUnresolved(host, + port))); } - host = host.substring(0, pidx); + } catch (QuorumPeerConfig.ConfigException exp) { + throw new IllegalArgumentException(exp); } - serverAddresses.add(InetSocketAddress.createUnresolved(host, port)); } } @@ -84,7 +108,7 @@ public String getChrootPath() { return chrootPath; } - public ArrayList getServerAddresses() { - return serverAddresses; + public Collection getServersCfg() { + return Collections.unmodifiableCollection(serverCfgList); } } diff --git a/src/java/main/org/apache/zookeeper/client/FourLetterWordMain.java b/src/java/main/org/apache/zookeeper/client/FourLetterWordMain.java index 19b45ba13e9..7a19020ab1b 100644 --- a/src/java/main/org/apache/zookeeper/client/FourLetterWordMain.java +++ b/src/java/main/org/apache/zookeeper/client/FourLetterWordMain.java @@ -33,6 +33,7 @@ import org.apache.zookeeper.common.X509Exception.SSLContextException; import org.apache.zookeeper.common.X509Util; +import org.apache.zookeeper.common.ZKConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,8 +47,8 @@ public class FourLetterWordMain { * @param port the destination port * @param cmd the 4letterword * @return server response - * @throws java.io.IOException - * @throws SSLContextException + * @throws java.io.IOException on error + * @throws SSLContextException on error */ public static String send4LetterWord(String host, int port, String cmd) throws IOException, SSLContextException { @@ -61,8 +62,8 @@ public static String send4LetterWord(String host, int port, String cmd) * @param cmd the 4letterword * @param secure whether to use SSL * @return server response - * @throws java.io.IOException - * @throws SSLContextException + * @throws java.io.IOException on error + * @throws SSLContextException on error */ public static String send4LetterWord(String host, int port, String cmd, boolean secure) throws IOException, SSLContextException { @@ -77,8 +78,8 @@ public static String send4LetterWord(String host, int port, String cmd, boolean * @param secure whether to use SSL * @param timeout in milliseconds, maximum time to wait while connecting/reading data * @return server response - * @throws java.io.IOException - * @throws SSLContextException + * @throws java.io.IOException on error + * @throws SSLContextException on error */ public static String send4LetterWord(String host, int port, String cmd, boolean secure, int timeout) throws IOException, SSLContextException { @@ -88,7 +89,10 @@ public static String send4LetterWord(String host, int port, String cmd, boolean new InetSocketAddress(InetAddress.getByName(null), port); if (secure) { LOG.info("using secure socket"); - SSLContext sslContext = X509Util.createSSLContext(); + // TODO: 4 letter main cannot be secure!, not supported yet. + // Which means this has to be local only. + SSLContext sslContext = X509Util.createSSLContext( + new ZKConfig(), null); SSLSocketFactory socketFactory = sslContext.getSocketFactory(); SSLSocket sslSock = (SSLSocket) socketFactory.createSocket(); sslSock.connect(hostaddress, timeout); diff --git a/src/java/main/org/apache/zookeeper/client/HostProvider.java b/src/java/main/org/apache/zookeeper/client/HostProvider.java index c47fd82a63e..618ea49af79 100644 --- a/src/java/main/org/apache/zookeeper/client/HostProvider.java +++ b/src/java/main/org/apache/zookeeper/client/HostProvider.java @@ -19,9 +19,10 @@ package org.apache.zookeeper.client; import java.net.InetSocketAddress; -import java.net.UnknownHostException; import java.util.Collection; +import org.apache.zookeeper.ServerCfg; + /** * A set of hosts a ZooKeeper client should connect to. * @@ -52,7 +53,7 @@ public interface HostProvider { * @param spinDelay * Milliseconds to wait if all hosts have been tried once. */ - public InetSocketAddress next(long spinDelay); + public ServerCfg next(long spinDelay); /** * Notify the HostProvider of a successful connection. @@ -63,10 +64,10 @@ public interface HostProvider { /** * Update the list of servers. This returns true if changing connections is necessary for load-balancing, false otherwise. - * @param serverAddresses new host list + * @param serversCfg new host list with optional SSL config. * @param currentHost the host to which this client is currently connected * @return true if changing connections is necessary for load-balancing, false otherwise */ - boolean updateServerList(Collection serverAddresses, - InetSocketAddress currentHost); + boolean updateServerList(final Collection serversCfg, + final InetSocketAddress currentHost); } diff --git a/src/java/main/org/apache/zookeeper/client/StaticHostProvider.java b/src/java/main/org/apache/zookeeper/client/StaticHostProvider.java index 9b856a2a1ce..bdba31b8ce2 100644 --- a/src/java/main/org/apache/zookeeper/client/StaticHostProvider.java +++ b/src/java/main/org/apache/zookeeper/client/StaticHostProvider.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Random; +import org.apache.zookeeper.ServerCfg; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,8 +39,7 @@ public final class StaticHostProvider implements HostProvider { private static final Logger LOG = LoggerFactory .getLogger(StaticHostProvider.class); - private List serverAddresses = new ArrayList( - 5); + private List serversCfg = new ArrayList(5); private Random sourceOfRandomness; private int lastIndex = -1; @@ -51,11 +51,9 @@ public final class StaticHostProvider implements HostProvider { */ private boolean reconfigMode = false; - private final List oldServers = new ArrayList( - 5); + private final List oldServers = new ArrayList(5); - private final List newServers = new ArrayList( - 5); + private final List newServers = new ArrayList(5); private int currentIndexOld = -1; private int currentIndexNew = -1; @@ -65,16 +63,18 @@ public final class StaticHostProvider implements HostProvider { /** * Constructs a SimpleHostSet. * - * @param serverAddresses - * possibly unresolved ZooKeeper server addresses + * @param serversCfg + * possibly unresolved ZooKeeper server addresses with + * optional ssl cfg. * @throws IllegalArgumentException * if serverAddresses is empty or resolves to an empty list */ - public StaticHostProvider(Collection serverAddresses) { + public StaticHostProvider( + final Collection serversCfg) { sourceOfRandomness = new Random(System.currentTimeMillis() ^ this.hashCode()); - this.serverAddresses = resolveAndShuffle(serverAddresses); - if (this.serverAddresses.isEmpty()) { + this.serversCfg = resolveAndShuffle(serversCfg); + if (this.serversCfg.isEmpty()) { throw new IllegalArgumentException( "A HostProvider may not be empty!"); } @@ -86,18 +86,19 @@ public StaticHostProvider(Collection serverAddresses) { * Constructs a SimpleHostSet. This constructor is used from StaticHostProviderTest to produce deterministic test results * by initializing sourceOfRandomness with the same seed * - * @param serverAddresses - * possibly unresolved ZooKeeper server addresses + * @param serversCfg + * possibly unresolved ZooKeeper server addresses with + * optional ssl cfg. * @param randomnessSeed a seed used to initialize sourceOfRandomnes * @throws IllegalArgumentException * if serverAddresses is empty or resolves to an empty list */ - public StaticHostProvider(Collection serverAddresses, + public StaticHostProvider(final Collection serversCfg, long randomnessSeed) { sourceOfRandomness = new Random(randomnessSeed); - this.serverAddresses = resolveAndShuffle(serverAddresses); - if (this.serverAddresses.isEmpty()) { + this.serversCfg = resolveAndShuffle(serversCfg); + if (this.serversCfg.isEmpty()) { throw new IllegalArgumentException( "A HostProvider may not be empty!"); } @@ -105,24 +106,35 @@ public StaticHostProvider(Collection serverAddresses, lastIndex = -1; } - private List resolveAndShuffle(Collection serverAddresses) { - List tmpList = new ArrayList(serverAddresses.size()); - for (InetSocketAddress address : serverAddresses) { + private List resolveAndShuffle( + final Collection serversCfg) { + final List tmpList = + new ArrayList(serversCfg.size()); + + for (final ServerCfg cfg : serversCfg) { try { - InetAddress ia = address.getAddress(); - String addr = (ia != null) ? ia.getHostAddress() : address.getHostString(); - InetAddress resolvedAddresses[] = InetAddress.getAllByName(addr); - for (InetAddress resolvedAddress : resolvedAddresses) { - InetAddress taddr = InetAddress.getByAddress(address.getHostString(), resolvedAddress.getAddress()); - tmpList.add(new InetSocketAddress(taddr, address.getPort())); + final InetAddress ia = cfg.getInetAddress().getAddress(); + String addr = (ia != null) ? ia.getHostAddress() : + cfg.getInetAddress().getHostString(); + final InetAddress resolvedAddresses[] = + InetAddress.getAllByName(addr); + for (final InetAddress resolvedAddress : resolvedAddresses) { + final InetAddress taddr = InetAddress.getByAddress( + cfg.getInetAddress().getHostString(), + resolvedAddress.getAddress()); + tmpList.add(new ServerCfg(cfg.getHostStr(), + new InetSocketAddress(taddr, + cfg.getInetAddress().getPort()), + cfg.getSslCertCfg())); } } catch (UnknownHostException ex) { - LOG.warn("No IP address found for server: {}", address, ex); + LOG.warn("No IP address found for server: {}", + cfg.getInetAddress(), ex); } } Collections.shuffle(tmpList, sourceOfRandomness); - return tmpList; - } + return Collections.unmodifiableList(tmpList); + } /** @@ -141,7 +153,7 @@ private List resolveAndShuffle(Collection * * See {@link https://issues.apache.org/jira/browse/ZOOKEEPER-1355} for the protocol and its evaluation, and * StaticHostProviderTest for the tests that illustrate how load balancing works with this policy. - * @param serverAddresses new host list + * @param serversCfg new host list with optional SSL config. * @param currentHost the host to which this client is currently connected * @return true if changing connections is necessary for load-balancing, false otherwise */ @@ -149,10 +161,10 @@ private List resolveAndShuffle(Collection @Override public synchronized boolean updateServerList( - Collection serverAddresses, - InetSocketAddress currentHost) { + final Collection serversCfg, + final InetSocketAddress currentHost) { // Resolve server addresses and shuffle them - List resolvedList = resolveAndShuffle(serverAddresses); + final List resolvedList = resolveAndShuffle(serversCfg); if (resolvedList.isEmpty()) { throw new IllegalArgumentException( "A HostProvider may not be empty!"); @@ -164,7 +176,7 @@ public synchronized boolean updateServerList( // choose "current" server according to the client rebalancing algorithm if (reconfigMode) { - myServer = next(0); + myServer = next(0).getInetAddress(); } // if the client is not currently connected to any server @@ -172,19 +184,21 @@ public synchronized boolean updateServerList( // reconfigMode = false (next shouldn't return null). if (lastIndex >= 0) { // take the last server to which we were connected - myServer = this.serverAddresses.get(lastIndex); + myServer = this.serversCfg.get(lastIndex).getInetAddress(); } else { // take the first server on the list - myServer = this.serverAddresses.get(0); + myServer = this.serversCfg.get(0).getInetAddress(); } } - for (InetSocketAddress addr : resolvedList) { - if (addr.getPort() == myServer.getPort() - && ((addr.getAddress() != null - && myServer.getAddress() != null && addr - .getAddress().equals(myServer.getAddress())) || addr - .getHostString().equals(myServer.getHostString()))) { + for (final ServerCfg serverCfg : resolvedList) { + if (serverCfg.getInetAddress().getPort() == myServer.getPort() + && ((serverCfg.getInetAddress().getAddress() != null + && myServer.getAddress() != null + && serverCfg.getInetAddress().getAddress().equals( + myServer.getAddress())) || + serverCfg.getInetAddress().getHostString().equals( + myServer.getHostString()))) { myServerInNewConfig = true; break; } @@ -196,11 +210,11 @@ public synchronized boolean updateServerList( oldServers.clear(); // Divide the new servers into oldServers that were in the previous list // and newServers that were not in the previous list - for (InetSocketAddress resolvedAddress : resolvedList) { - if (this.serverAddresses.contains(resolvedAddress)) { - oldServers.add(resolvedAddress); + for (final ServerCfg resolvedCfg : resolvedList) { + if (this.serversCfg.contains(resolvedCfg)) { + oldServers.add(resolvedCfg); } else { - newServers.add(resolvedAddress); + newServers.add(resolvedCfg); } } @@ -208,13 +222,13 @@ public synchronized boolean updateServerList( int numNew = newServers.size(); // number of servers increased - if (numOld + numNew > this.serverAddresses.size()) { + if (numOld + numNew > this.serversCfg.size()) { if (myServerInNewConfig) { // my server is in new config, but load should be decreased. // Need to decide if this client // is moving to one of the new servers - if (sourceOfRandomness.nextFloat() <= (1 - ((float) this.serverAddresses - .size()) / (numOld + numNew))) { + if (sourceOfRandomness.nextFloat() <= (1 - + ((float) this.serversCfg.size()) / (numOld + numNew))) { pNew = 1; pOld = 0; } else { @@ -234,8 +248,10 @@ public synchronized boolean updateServerList( // stay with this server and do nothing special reconfigMode = false; } else { - pOld = ((float) (numOld * (this.serverAddresses.size() - (numOld + numNew)))) - / ((numOld + numNew) * (this.serverAddresses.size() - numOld)); + pOld = ((float) (numOld * (this.serversCfg.size() - + (numOld + numNew)))) + / ((numOld + numNew) * + (this.serversCfg.size() - numOld)); pNew = 1 - pOld; } } @@ -245,24 +261,24 @@ public synchronized boolean updateServerList( } else { currentIndex = -1; } - this.serverAddresses = resolvedList; + this.serversCfg = resolvedList; currentIndexOld = -1; currentIndexNew = -1; lastIndex = currentIndex; return reconfigMode; } - public synchronized InetSocketAddress getServerAtIndex(int i) { - if (i < 0 || i >= serverAddresses.size()) return null; - return serverAddresses.get(i); + public synchronized ServerCfg getServerAtIndex(int i) { + if (i < 0 || i >= serversCfg.size()) return null; + return serversCfg.get(i); } - public synchronized InetSocketAddress getServerAtCurrentIndex() { + public synchronized ServerCfg getServerAtCurrentIndex() { return getServerAtIndex(currentIndex); } public synchronized int size() { - return serverAddresses.size(); + return serversCfg.size(); } /** @@ -278,7 +294,7 @@ public synchronized int size() { * * When called, this should be protected by synchronized(this) */ - private InetSocketAddress nextHostInReconfigMode() { + private ServerCfg nextHostInReconfigMode() { boolean takeNew = (sourceOfRandomness.nextFloat() <= pNew); // take one of the new servers if it is possible (there are still such @@ -301,26 +317,27 @@ private InetSocketAddress nextHostInReconfigMode() { return null; } - public InetSocketAddress next(long spinDelay) { + @Override + public ServerCfg next(long spinDelay) { boolean needToSleep = false; - InetSocketAddress addr; + ServerCfg serverCfg; synchronized(this) { if (reconfigMode) { - addr = nextHostInReconfigMode(); - if (addr != null) { - currentIndex = serverAddresses.indexOf(addr); - return addr; + serverCfg = nextHostInReconfigMode(); + if (serverCfg != null) { + currentIndex = serversCfg.indexOf(serverCfg); + return serverCfg; } //tried all servers and couldn't connect reconfigMode = false; needToSleep = (spinDelay > 0); } ++currentIndex; - if (currentIndex == serverAddresses.size()) { + if (currentIndex == serversCfg.size()) { currentIndex = 0; } - addr = serverAddresses.get(currentIndex); + serverCfg = serversCfg.get(currentIndex); needToSleep = needToSleep || (currentIndex == lastIndex && spinDelay > 0); if (lastIndex == -1) { // We don't want to sleep on the first ever connect attempt. @@ -335,7 +352,7 @@ public InetSocketAddress next(long spinDelay) { } } - return addr; + return serverCfg; } public synchronized void onConnected() { diff --git a/src/java/main/org/apache/zookeeper/common/X509Exception.java b/src/java/main/org/apache/zookeeper/common/X509Exception.java index 984a2abad27..85f11e78ed4 100644 --- a/src/java/main/org/apache/zookeeper/common/X509Exception.java +++ b/src/java/main/org/apache/zookeeper/common/X509Exception.java @@ -49,6 +49,10 @@ public TrustManagerException(String message) { public TrustManagerException(Throwable cause) { super(cause); } + + public TrustManagerException(String message, Throwable cause) { + super(message, cause); + } } public static class SSLContextException extends X509Exception { diff --git a/src/java/main/org/apache/zookeeper/common/X509Util.java b/src/java/main/org/apache/zookeeper/common/X509Util.java index c48d6941a06..c34af092ddb 100644 --- a/src/java/main/org/apache/zookeeper/common/X509Util.java +++ b/src/java/main/org/apache/zookeeper/common/X509Util.java @@ -17,6 +17,22 @@ */ package org.apache.zookeeper.common; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.security.InvalidKeyException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.PublicKey; +import java.security.SignatureException; +import java.security.UnrecoverableKeyException; +import java.security.cert.CertificateEncodingException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; import javax.net.ssl.KeyManager; import javax.net.ssl.KeyManagerFactory; @@ -25,14 +41,17 @@ import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509KeyManager; import javax.net.ssl.X509TrustManager; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.security.KeyStore; +import javax.xml.bind.DatatypeConverter; +import org.apache.zookeeper.SSLCertCfg; +import org.apache.zookeeper.server.quorum.QuorumPeer; +import org.apache.zookeeper.server.quorum.util.ZKDynamicX509TrustManager; +import org.apache.zookeeper.server.quorum.util.ZKPeerX509TrustManager; +import org.apache.zookeeper.server.quorum.util.ZKX509TrustManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static javax.xml.bind.DatatypeConverter.printHexBinary; import static org.apache.zookeeper.common.X509Exception.KeyManagerException; import static org.apache.zookeeper.common.X509Exception.SSLContextException; import static org.apache.zookeeper.common.X509Exception.TrustManagerException; @@ -43,6 +62,11 @@ public class X509Util { private static final Logger LOG = LoggerFactory.getLogger(X509Util.class); + /** + * TODO: XXX: Move to ZKClient + */ + public static final String SSL_VERSION_DEFAULT = "TLSv1"; + public static final String SSL_VERSION = "zookeeper.ssl.version"; /** * @deprecated Use {@link ZKConfig#SSL_KEYSTORE_LOCATION} * instead. @@ -74,66 +98,156 @@ public class X509Util { @Deprecated public static final String SSL_AUTHPROVIDER = "zookeeper.ssl.authProvider"; - public static SSLContext createSSLContext() throws SSLContextException { - /** - * Since Configuration initializes the key store and trust store related - * configuration from system property. Reading property from - * configuration will be same reading from system property - */ - ZKConfig config=new ZKConfig(); - return createSSLContext(config); + /** + * TODO: XXX: Move to ZKClient + */ + public static final String SSL_TRUSTSTORE_CA_ALIAS = + "zookeeper.ssl.trustStore.rootCA.alias"; + public static final String SSL_DIGEST_DEFAULT_ALGO ="SHA-256"; + public static final String SSL_DIGEST_ALGOS = "quorum.ssl.digest.algos"; + + /** + * API to create SSL context for clients. Supports both self-signed and + * CA signed. + * @param peerAddr host that client is trying to connect to + * @param peerCertCfg host's self-signed cert of CA signed cert. + * @return SSLContext with right verification based on self-signed or CA + * signed. + * @throws SSLContextException + */ + public static SSLContext createSSLContext( + final ZKConfig config, + final InetSocketAddress peerAddr, + final SSLCertCfg peerCertCfg) + throws SSLContextException { + final KeyManager[] keyManagers = createKeyManagers(config); + TrustManager[] trustManagers; + if (peerCertCfg.isSelfSigned()) { + trustManagers = createTrustManagers(config, peerAddr, + peerCertCfg.getCertFingerPrintStr()); + } else if (peerCertCfg.isCASigned()) { + // Lets load the CA for truststore. + trustManagers = createTrustManagers(config, null); + } else { + throw new IllegalArgumentException("Invalid argument, no SSL cfg " + + "provided"); + } + + return createSSLContext(config, keyManagers, trustManagers); } - public static SSLContext createSSLContext(ZKConfig config) throws SSLContextException { - KeyManager[] keyManagers = null; - TrustManager[] trustManagers = null; + /** + * SSL context which can be used by both client and server side which + * depend on dynamic config for authentication. Hence we need quorumPeer. + * @param quorumPeer Used for getting QuorumVerifier and certs from + * QuorumPeerConfig. Both commited and last verified. + * @return SSLContext which can perform authentication based on dynamic cfg. + * @throws SSLContextException + */ + public static SSLContext createSSLContext(final ZKConfig config, + final QuorumPeer quorumPeer) + throws SSLContextException { + final KeyManager[] keyManagers = createKeyManagers(config); + final TrustManager[] trustManagers = + createTrustManagers(config, quorumPeer); - String keyStoreLocationProp = config.getProperty(ZKConfig.SSL_KEYSTORE_LOCATION); - String keyStorePasswordProp = config.getProperty(ZKConfig.SSL_KEYSTORE_PASSWD); + return createSSLContext(config, keyManagers, trustManagers); + } - // There are legal states in some use cases for null KeyManager or TrustManager. - // But if a user wanna specify one, location and password are required. + private static KeyManager[] createKeyManagers(final ZKConfig config) throws + SSLContextException { + final String keyStoreLocationProp = + config.getProperty(SSL_KEYSTORE_LOCATION); + final String keyStorePasswordProp = + config.getProperty(SSL_KEYSTORE_PASSWD); if (keyStoreLocationProp == null && keyStorePasswordProp == null) { LOG.warn("keystore not specified for client connection"); + return null; } else { if (keyStoreLocationProp == null) { - throw new SSLContextException("keystore location not specified for client connection"); + throw new SSLContextException("keystore location not " + + "specified for client connection"); } if (keyStorePasswordProp == null) { - throw new SSLContextException("keystore password not specified for client connection"); + throw new SSLContextException("keystore password not " + + "specified for client connection"); } try { - keyManagers = new KeyManager[]{ - createKeyManager(keyStoreLocationProp, keyStorePasswordProp)}; + return new KeyManager[]{ + createKeyManager(keyStoreLocationProp, + keyStorePasswordProp)}; } catch (KeyManagerException e) { throw new SSLContextException("Failed to create KeyManager", e); } } + } - String trustStoreLocationProp = config.getProperty(ZKConfig.SSL_TRUSTSTORE_LOCATION); - String trustStorePasswordProp = config.getProperty(ZKConfig.SSL_TRUSTSTORE_PASSWD); + /** + * If QuorumPeer is not provided and this is called it means we are CA + * mode and need both truststore location and password. + * @param quorumPeer + * @return + * @throws SSLContextException + */ + private static TrustManager[] createTrustManagers( + final ZKConfig config, + final QuorumPeer quorumPeer) throws SSLContextException { + String trustStoreLocationProp = + config.getProperty(SSL_TRUSTSTORE_LOCATION); + String trustStorePasswordProp = + config.getProperty(SSL_TRUSTSTORE_PASSWD); if (trustStoreLocationProp == null && trustStorePasswordProp == null) { - LOG.warn("keystore not specified for client connection"); + if (quorumPeer == null) { + final String errStr = "truststore not specified"; + LOG.error(errStr); + throw new SSLContextException(errStr); + } + + // Create self-signed verification using QuorumPeer. + return new TrustManager[] {createTrustManager(quorumPeer)}; } else { if (trustStoreLocationProp == null) { - throw new SSLContextException("keystore location not specified for client connection"); + throw new SSLContextException("truststore location not " + + "specified for client connection"); } if (trustStorePasswordProp == null) { - throw new SSLContextException("keystore password not specified for client connection"); + throw new SSLContextException("truststore password not " + + "specified for client connection"); } try { - trustManagers = new TrustManager[]{ - createTrustManager(trustStoreLocationProp, trustStorePasswordProp)}; + return new TrustManager[] { + createTrustManager(config, trustStoreLocationProp, + trustStorePasswordProp)}; } catch (TrustManagerException e) { - throw new SSLContextException("Failed to create KeyManager", e); + throw new SSLContextException( + "Failed to create TrustManager", e); } } + } + + private static TrustManager[] createTrustManagers( + final ZKConfig config, + final InetSocketAddress peerAddr, + final String peerCertFingerPrintStr) { + return new TrustManager[]{ + createTrustManager(config, peerAddr, + peerCertFingerPrintStr)}; + } - SSLContext sslContext = null; + private static SSLContext createSSLContext( + final ZKConfig config, + final KeyManager[] keyManagers, + final TrustManager[] trustManagers) + throws SSLContextException { + String sslVersion = config.getProperty(SSL_VERSION); + if (sslVersion == null) { + sslVersion = SSL_VERSION_DEFAULT; + } + SSLContext sslContext; try { - sslContext = SSLContext.getInstance("TLSv1"); + sslContext = SSLContext.getInstance(sslVersion); sslContext.init(keyManagers, trustManagers, null); } catch (Exception e) { throw new SSLContextException(e); @@ -141,17 +255,14 @@ public static SSLContext createSSLContext(ZKConfig config) throws SSLContextExce return sslContext; } - public static X509KeyManager createKeyManager(String keyStoreLocation, String keyStorePassword) + public static X509KeyManager createKeyManager( + final String keyStoreLocation, final String keyStorePassword) throws KeyManagerException { - FileInputStream inputStream = null; try { - char[] keyStorePasswordChars = keyStorePassword.toCharArray(); - File keyStoreFile = new File(keyStoreLocation); - KeyStore ks = KeyStore.getInstance("JKS"); - inputStream = new FileInputStream(keyStoreFile); - ks.load(inputStream, keyStorePasswordChars); + + KeyStore ks = loadKeyStore(keyStoreLocation, keyStorePassword); KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509"); - kmf.init(ks, keyStorePasswordChars); + kmf.init(ks, keyStorePassword.toCharArray()); for (KeyManager km : kmf.getKeyManagers()) { if (km instanceof X509KeyManager) { @@ -160,43 +271,227 @@ public static X509KeyManager createKeyManager(String keyStoreLocation, String ke } throw new KeyManagerException("Couldn't find X509KeyManager"); - } catch (Exception e) { + } catch (KeyStoreException | NoSuchAlgorithmException | + CertificateException | UnrecoverableKeyException | + IOException e) { throw new KeyManagerException(e); - } finally { - if (inputStream != null) { - try { - inputStream.close(); - } catch (IOException e) {} - } } } - public static X509TrustManager createTrustManager(String trustStoreLocation, String trustStorePassword) - throws TrustManagerException { - FileInputStream inputStream = null; - try { - char[] trustStorePasswordChars = trustStorePassword.toCharArray(); - File trustStoreFile = new File(trustStoreLocation); - KeyStore ts = KeyStore.getInstance("JKS"); - inputStream = new FileInputStream(trustStoreFile); - ts.load(inputStream, trustStorePasswordChars); - TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509"); - tmf.init(ts); - - for (TrustManager tm : tmf.getTrustManagers()) { - if (tm instanceof X509TrustManager) { - return (X509TrustManager) tm; + private static KeyStore loadKeyStore(final String keyStoreLocation, + final String keyStorePassword) + throws KeyStoreException, NoSuchAlgorithmException, + CertificateException, IOException { + char[] keyStorePasswordChars = keyStorePassword.toCharArray(); + File keyStoreFile = new File(keyStoreLocation); + KeyStore ks = KeyStore.getInstance("JKS"); + try (final FileInputStream inputStream = new FileInputStream + (keyStoreFile)) { + ks.load(inputStream, keyStorePasswordChars); + } + return ks; + } + + private static X509TrustManager createTrustManager( + final ZKConfig config, + final InetSocketAddress peerAddr, + final String peerCertFingerPrintStr) { + return new ZKPeerX509TrustManager(peerAddr, peerCertFingerPrintStr); + } + + public static X509TrustManager createTrustManager( + final ZKConfig config, + final String trustStoreLocation, final String trustStorePassword) + throws TrustManagerException, SSLContextException { + String trustStoreCAAlias = + config.getProperty(SSL_TRUSTSTORE_CA_ALIAS); + if (trustStoreCAAlias == null) { + final String errStr = "No CA Alias provided, need one to work in " + + "CA mode."; + LOG.error(errStr); + throw new TrustManagerException(errStr); + } + try(final FileInputStream inputStream = + new FileInputStream(new File(trustStoreLocation))) { + char[] trustStorePasswordChars = + trustStorePassword.toCharArray(); + KeyStore ts = KeyStore.getInstance("JKS"); + ts.load(inputStream, trustStorePasswordChars); + TrustManagerFactory tmf = + TrustManagerFactory.getInstance("SunX509"); + tmf.init(ts); + X509Certificate rootCA = + getCertWithAlias(ts, trustStoreCAAlias); + if (rootCA == null) { + final String str = "failed to find root CA from: " + + trustStoreLocation + " with alias: " + + trustStoreCAAlias; + LOG.error(str); + throw new TrustManagerException(str); } + + return createTrustManager(rootCA); + } catch (IOException | KeyStoreException | NoSuchAlgorithmException + | CertificateException e) { + final String errStr = "Could not load truststore: " + + trustStoreLocation; + LOG.error("{}", errStr, e); + throw new TrustManagerException(errStr, e); + } + } + + private static X509TrustManager createTrustManager( + final X509Certificate rootCA) { + return new ZKX509TrustManager(rootCA); + } + + private static X509TrustManager createTrustManager( + final QuorumPeer quorumPeer) { + return new ZKDynamicX509TrustManager(quorumPeer); + } + private static X509Certificate getCertWithAlias( + final KeyStore trustStore, final String alias) + throws KeyStoreException { + X509Certificate cert; + try { + cert = (X509Certificate) trustStore.getCertificate(alias); + } catch (KeyStoreException exp) { + LOG.error("failed to load CA cert, exp: " + exp); + throw exp; + } + + return cert; + } + + /** + * Parse parsed system property and find a valid algo that matches + * the finger print passed. Will return null if it couldn't + * @param fingerPrint + * @return MessageDigest object, null on error. + */ + public static MessageDigest getSupportedMessageDigestForFpStr( + final String fingerPrint) { + final String[] algos = getConfigureDigestAlgos(); + String validAlgo = null; + + for (int i = 0; i < algos.length; i++) { + LOG.info("Trying available algo: " + algos[i]); + if (fingerPrint.toLowerCase().startsWith(algos[i])) { + validAlgo = algos[i]; + break; } - throw new TrustManagerException("Couldn't find X509TrustManager"); - } catch (Exception e) { - throw new TrustManagerException(e); - } finally { - if (inputStream != null) { - try { - inputStream.close(); - } catch (IOException e) {} - } + } + + // If there is no valid algo then return null + if (validAlgo == null) { + LOG.error("Could not find valid algo str in fingerprint: " + + fingerPrint); + return null; + } + + MessageDigest md = getMessageDigestByAlgo(validAlgo); + if (md == null) { + return null; + } + + // Validate that given input matches expected length for + // the supported algorithm + final String fp = fingerPrint.trim().toUpperCase() + .replace(md.getAlgorithm(), "") + .replace("-", "").toLowerCase(); + + byte[] b = DatatypeConverter.parseHexBinary(fp); + if (b.length != md.getDigestLength()) { + LOG.error("Invalid digest, length mismatch for fingerprint: " + + fingerPrint + "has length: " + b.length + "algo: " + + md.getAlgorithm() + " needs length: " + + md.getDigestLength()); + return null; + } + LOG.info("given str fp: " + fp + ", got fp: " + + printHexBinary(b)); + return md; + } + + private static String[] getConfigureDigestAlgos() { + String digest_algos = System.getProperty(SSL_DIGEST_ALGOS); + if (digest_algos == null) { + digest_algos = SSL_DIGEST_DEFAULT_ALGO; + } + + return digest_algos.trim().toLowerCase().split(","); + } + + private static MessageDigest getMessageDigestByAlgo( + final String validAlgo) { + MessageDigest md = null; + try { + LOG.info("Valid algo: " + validAlgo); + md = MessageDigest.getInstance(validAlgo.toUpperCase()); + } catch (NoSuchAlgorithmException e) { + LOG.error("Invalid algo: " + validAlgo + " support algos: " + + getConfigureDigestAlgos()); + } + + return md; + } + + /** + * Get the right MessageDigest i.e only if it is configured and validate + * the cert with the given finger print. + * @param fingerPrint + * @param cert + * @return True on success + * @throws CertificateEncodingException + */ + public static boolean validateCert(final String fingerPrint, + final X509Certificate cert) + throws CertificateEncodingException, NoSuchAlgorithmException { + final MessageDigest fpMsgDigest = + getSupportedMessageDigestForFpStr(fingerPrint); + if (fpMsgDigest == null) { + return false; + } + + return validateCert(fpMsgDigest, fingerPrint, cert); + } + + public static boolean validateCert(final MessageDigest messageDigest, + final String fingerPrintStr, + final X509Certificate cert) + throws CertificateEncodingException, NoSuchAlgorithmException { + return fingerPrintStr.toLowerCase().equals( + SSLCertCfg.getDigestToCertFp( + getMessageDigestFromCert(cert, + messageDigest.getAlgorithm())).toLowerCase()); + } + + public static MessageDigest getMessageDigestFromCert( + final X509Certificate cert, final String messageDigestAlgo) + throws NoSuchAlgorithmException, CertificateEncodingException { + final MessageDigest certMsgDigest = + MessageDigest.getInstance(messageDigestAlgo); + + certMsgDigest.update(cert.getEncoded()); + return certMsgDigest; + } + + /** + * Checks whether given X.509 certificate is self-signed. + */ + public static boolean verifySelfSigned(X509Certificate cert) + throws CertificateException { + try { + // Try to verify certificate signature with its own public key + final PublicKey key = cert.getPublicKey(); + cert.verify(key); + return true; + } catch (InvalidKeyException | SignatureException | + NoSuchAlgorithmException | NoSuchProviderException exp) { + // Invalid signature --> not self-signed + final String errStr = "Invalid not self-signed"; + LOG.error("{}", errStr, exp); + throw new CertificateException(exp); } } -} \ No newline at end of file +} diff --git a/src/java/main/org/apache/zookeeper/server/NettyServerCnxnFactory.java b/src/java/main/org/apache/zookeeper/server/NettyServerCnxnFactory.java index 25b682b8633..0b7aaed19dc 100644 --- a/src/java/main/org/apache/zookeeper/server/NettyServerCnxnFactory.java +++ b/src/java/main/org/apache/zookeeper/server/NettyServerCnxnFactory.java @@ -18,8 +18,6 @@ package org.apache.zookeeper.server; -import static org.jboss.netty.buffer.ChannelBuffers.dynamicBuffer; - import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -35,14 +33,11 @@ import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLPeerUnverifiedException; import javax.net.ssl.SSLSession; -import javax.net.ssl.X509KeyManager; -import javax.net.ssl.X509TrustManager; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.common.X509Exception; -import org.apache.zookeeper.common.X509Exception.SSLContextException; import org.apache.zookeeper.common.X509Util; +import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.server.auth.ProviderRegistry; import org.apache.zookeeper.server.auth.X509AuthenticationProvider; import org.jboss.netty.bootstrap.ServerBootstrap; @@ -68,6 +63,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.jboss.netty.buffer.ChannelBuffers.dynamicBuffer; + public class NettyServerCnxnFactory extends ServerCnxnFactory { private static final Logger LOG = LoggerFactory.getLogger(NettyServerCnxnFactory.class); @@ -109,14 +106,8 @@ public void channelConnected(ChannelHandlerContext ctx, zkServer, NettyServerCnxnFactory.this); ctx.setAttachment(cnxn); - if (secure) { - SslHandler sslHandler = ctx.getPipeline().get(SslHandler.class); - ChannelFuture handshakeFuture = sslHandler.handshake(); - handshakeFuture.addListener(new CertificateVerifier(sslHandler, cnxn)); - } else { - allChannels.add(ctx.getChannel()); - addCnxn(cnxn); - } + allChannels.add(ctx.getChannel()); + addCnxn(cnxn); } @Override @@ -355,8 +346,11 @@ private synchronized void initSSL(ChannelPipeline p) String authProviderProp = System.getProperty(ZKConfig.SSL_AUTHPROVIDER); SSLContext sslContext; if (authProviderProp == null) { - sslContext = X509Util.createSSLContext(); + sslContext = X509Util.createSSLContext(new ZKConfig(), quorumPeer); } else { + throw new IllegalAccessError("No support for auth provider: " + + authProviderProp); + /* sslContext = SSLContext.getInstance("TLSv1"); X509AuthenticationProvider authProvider = (X509AuthenticationProvider)ProviderRegistry.getProvider( @@ -374,6 +368,7 @@ private synchronized void initSSL(ChannelPipeline p) sslContext.init(new X509KeyManager[] { authProvider.getKeyManager() }, new X509TrustManager[] { authProvider.getTrustManager() }, null); + */ } SSLEngine sslEngine = sslContext.createSSLEngine(); diff --git a/src/java/main/org/apache/zookeeper/server/ServerCnxnFactory.java b/src/java/main/org/apache/zookeeper/server/ServerCnxnFactory.java index fc6766cdc13..1f5053b7898 100644 --- a/src/java/main/org/apache/zookeeper/server/ServerCnxnFactory.java +++ b/src/java/main/org/apache/zookeeper/server/ServerCnxnFactory.java @@ -22,8 +22,8 @@ import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.Collections; -import java.util.Set; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import javax.management.JMException; @@ -36,6 +36,7 @@ import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.jmx.MBeanRegistry; import org.apache.zookeeper.server.auth.SaslServerCallbackHandler; +import org.apache.zookeeper.server.quorum.QuorumPeer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,6 +49,7 @@ public abstract class ServerCnxnFactory { // Tells whether SSL is enabled on this ServerCnxnFactory protected boolean secure; + protected QuorumPeer quorumPeer; /** * The buffer will cause the connection to be close when we do a send. @@ -79,6 +81,10 @@ public void configure(InetSocketAddress addr, int maxcc) throws IOException { public abstract void configure(InetSocketAddress addr, int maxcc, boolean secure) throws IOException; + public void setQuorumPeer(QuorumPeer quorumPeer) { + this.quorumPeer = quorumPeer; + } + public abstract void reconfigure(InetSocketAddress addr); protected SaslServerCallbackHandler saslServerCallbackHandler; diff --git a/src/java/main/org/apache/zookeeper/server/auth/X509AuthenticationProvider.java b/src/java/main/org/apache/zookeeper/server/auth/X509AuthenticationProvider.java index 902b3078710..f6a1174d8c6 100644 --- a/src/java/main/org/apache/zookeeper/server/auth/X509AuthenticationProvider.java +++ b/src/java/main/org/apache/zookeeper/server/auth/X509AuthenticationProvider.java @@ -26,10 +26,11 @@ import javax.security.auth.x500.X500Principal; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.common.ZKConfig; +import org.apache.zookeeper.common.X509Exception; import org.apache.zookeeper.common.X509Exception.KeyManagerException; import org.apache.zookeeper.common.X509Exception.TrustManagerException; import org.apache.zookeeper.common.X509Util; +import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.data.Id; import org.apache.zookeeper.server.ServerCnxn; import org.slf4j.Logger; @@ -85,10 +86,11 @@ public X509AuthenticationProvider() { ZKConfig.SSL_TRUSTSTORE_PASSWD); try { - tm = X509Util.createTrustManager( + tm = X509Util.createTrustManager(new ZKConfig(), trustStoreLocationProp, trustStorePasswordProp); - } catch (TrustManagerException e) { + } catch (TrustManagerException | X509Exception.SSLContextException e) { LOG.error("Failed to create trust manager", e); + throw new IllegalAccessError("Failed to create trust manager"); } this.keyManager = km; diff --git a/src/java/main/org/apache/zookeeper/server/quorum/Follower.java b/src/java/main/org/apache/zookeeper/server/quorum/Follower.java index 1cdc2025c7e..a6edb779f03 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/Follower.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/Follower.java @@ -19,10 +19,10 @@ package org.apache.zookeeper.server.quorum; import java.io.IOException; -import java.net.InetSocketAddress; import java.nio.ByteBuffer; import org.apache.jute.Record; +import org.apache.zookeeper.ServerCfg; import org.apache.zookeeper.ZooDefs.OpCode; import org.apache.zookeeper.common.Time; import org.apache.zookeeper.server.Request; @@ -31,6 +31,7 @@ import org.apache.zookeeper.server.util.ZxidUtils; import org.apache.zookeeper.txn.SetDataTxn; import org.apache.zookeeper.txn.TxnHeader; + /** * This class has the control logic for the Follower. */ @@ -71,9 +72,9 @@ void followLeader() throws InterruptedException { self.end_fle = 0; fzk.registerJMX(new FollowerBean(this, zk), self.jmxLocalPeerBean); try { - InetSocketAddress addr = findLeader(); + final ServerCfg serverCfg = findLeader(); try { - connectToLeader(addr); + connectToLeader(serverCfg); long newEpochZxid = registerWithLeader(Leader.FOLLOWERINFO); if (self.isReconfigStateChange()) throw new Exception("learned about role change"); diff --git a/src/java/main/org/apache/zookeeper/server/quorum/Leader.java b/src/java/main/org/apache/zookeeper/server/quorum/Leader.java index 13f7ec97183..e977ff19f80 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/Leader.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/Leader.java @@ -20,7 +20,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.net.BindException; import java.net.ServerSocket; import java.net.Socket; import java.net.SocketAddress; @@ -33,14 +32,15 @@ import java.util.Iterator; import java.util.List; import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; import org.apache.jute.BinaryOutputArchive; import org.apache.zookeeper.ZooDefs.OpCode; import org.apache.zookeeper.common.Time; +import org.apache.zookeeper.common.X509Exception; import org.apache.zookeeper.server.FinalRequestProcessor; import org.apache.zookeeper.server.Request; import org.apache.zookeeper.server.RequestProcessor; @@ -51,7 +51,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * This class has the control logic for the Leader. */ @@ -220,21 +219,21 @@ public boolean isQuorumSynced(QuorumVerifier qv) { this.self = self; try { if (self.getQuorumListenOnAllIPs()) { - ss = new ServerSocket(self.getQuorumAddress().getPort()); + ss = this.self.socketFactory.buildForServer(self, + self.getQuorumAddress().getPort()); } else { - ss = new ServerSocket(); + ss = this.self.socketFactory.buildForServer(self, + self.getQuorumAddress().getPort(), + self.getQuorumAddress().getAddress()); } ss.setReuseAddress(true); - if (!self.getQuorumListenOnAllIPs()) { - ss.bind(self.getQuorumAddress()); - } - } catch (BindException e) { + } catch (IOException | X509Exception e) { if (self.getQuorumListenOnAllIPs()) { LOG.error("Couldn't bind to port " + self.getQuorumAddress().getPort(), e); } else { LOG.error("Couldn't bind to " + self.getQuorumAddress(), e); } - throw e; + throw new IOException(e); } this.zk = zk; } diff --git a/src/java/main/org/apache/zookeeper/server/quorum/Learner.java b/src/java/main/org/apache/zookeeper/server/quorum/Learner.java index f048da8a4d9..77620d0ed58 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/Learner.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/Learner.java @@ -38,9 +38,9 @@ import org.apache.jute.InputArchive; import org.apache.jute.OutputArchive; import org.apache.jute.Record; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.zookeeper.ServerCfg; import org.apache.zookeeper.ZooDefs.OpCode; +import org.apache.zookeeper.common.X509Exception; import org.apache.zookeeper.server.Request; import org.apache.zookeeper.server.ServerCnxn; import org.apache.zookeeper.server.ZooTrace; @@ -50,6 +50,8 @@ import org.apache.zookeeper.server.util.ZxidUtils; import org.apache.zookeeper.txn.SetDataTxn; import org.apache.zookeeper.txn.TxnHeader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class is the superclass of two of the three main actors in a ZK @@ -193,21 +195,22 @@ void request(Request request) throws IOException { /** * Returns the address of the node we think is the leader. */ - protected InetSocketAddress findLeader() { - InetSocketAddress addr = null; + protected ServerCfg findLeader() { + ServerCfg serverCfg = null; // Find the leader by id Vote current = self.getCurrentVote(); for (QuorumServer s : self.getView().values()) { if (s.id == current.getId()) { - addr = s.addr; + serverCfg = new ServerCfg(s.addr.getHostString(), s.addr, + s.getSslCertCfg()); break; } } - if (addr == null) { + if (serverCfg == null) { LOG.warn("Couldn't find the leader with id = " + current.getId()); } - return addr; + return serverCfg; } /** @@ -230,14 +233,15 @@ protected void sockConnect(Socket sock, InetSocketAddress addr, int timeout) /** * Establish a connection with the Leader found by findLeader. Retries * until either initLimit time has elapsed or 5 tries have happened. - * @param addr - the address of the Leader to connect to. + * @param serverCfg - the address of the Leader to connect to with SSL cfg. * @throws IOException - if the socket connection fails on the 5th attempt * @throws ConnectException * @throws InterruptedException */ - protected void connectToLeader(InetSocketAddress addr) - throws IOException, ConnectException, InterruptedException { - sock = new Socket(); + protected void connectToLeader(final ServerCfg serverCfg) + throws IOException, ConnectException, InterruptedException, X509Exception { + sock = this.self.socketFactory.buildForClient( + serverCfg.getInetAddress(), serverCfg.getSslCertCfg()); sock.setSoTimeout(self.tickTime * self.initLimit); int initLimitTime = self.tickTime * self.initLimit; @@ -253,7 +257,9 @@ protected void connectToLeader(InetSocketAddress addr) throw new IOException("initLimit exceeded on retries."); } - sockConnect(sock, addr, Math.min(self.tickTime * self.syncLimit, remainingInitLimitTime)); + sockConnect(sock, serverCfg.getInetAddress(), + Math.min(self.tickTime * self.syncLimit, + remainingInitLimitTime)); sock.setTcpNoDelay(nodelay); break; } catch (IOException e) { @@ -262,18 +268,22 @@ protected void connectToLeader(InetSocketAddress addr) if (remainingInitLimitTime <= 1000) { LOG.error("Unexpected exception, initLimit exceeded. tries=" + tries + ", remaining init limit=" + remainingInitLimitTime + - ", connecting to " + addr,e); + ", connecting to " + + serverCfg.getInetAddress(), e); throw e; } else if (tries >= 4) { LOG.error("Unexpected exception, retries exceeded. tries=" + tries + ", remaining init limit=" + remainingInitLimitTime + - ", connecting to " + addr,e); + ", connecting to " + + serverCfg.getInetAddress(), e); throw e; } else { LOG.warn("Unexpected exception, tries=" + tries + ", remaining init limit=" + remainingInitLimitTime + - ", connecting to " + addr,e); - sock = new Socket(); + ", connecting to " + serverCfg.getInetAddress(), e); + sock = this.self.socketFactory.buildForClient( + serverCfg.getInetAddress(), + serverCfg.getSslCertCfg()); sock.setSoTimeout(self.tickTime * self.initLimit); } } @@ -283,7 +293,7 @@ protected void connectToLeader(InetSocketAddress addr) sock.getInputStream())); bufferedOutput = new BufferedOutputStream(sock.getOutputStream()); leaderOs = BinaryOutputArchive.getArchive(bufferedOutput); - } + } /** * Once connected to the leader, perform the handshake protocol to diff --git a/src/java/main/org/apache/zookeeper/server/quorum/Observer.java b/src/java/main/org/apache/zookeeper/server/quorum/Observer.java index 27368c7527f..95e0c6eabfc 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/Observer.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/Observer.java @@ -19,10 +19,10 @@ package org.apache.zookeeper.server.quorum; import java.io.IOException; -import java.net.InetSocketAddress; import java.nio.ByteBuffer; import org.apache.jute.Record; +import org.apache.zookeeper.ServerCfg; import org.apache.zookeeper.server.ObserverBean; import org.apache.zookeeper.server.Request; import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; @@ -63,10 +63,10 @@ void observeLeader() throws Exception { zk.registerJMX(new ObserverBean(this, zk), self.jmxLocalPeerBean); try { - InetSocketAddress addr = findLeader(); - LOG.info("Observing " + addr); + final ServerCfg serverCfg = findLeader(); + LOG.info("Observing " + serverCfg.getInetAddress()); try { - connectToLeader(addr); + connectToLeader(serverCfg); long newLeaderZxid = registerWithLeader(Leader.OBSERVERINFO); if (self.isReconfigStateChange()) throw new Exception("learned about role change"); diff --git a/src/java/main/org/apache/zookeeper/server/quorum/QuorumCnxManager.java b/src/java/main/org/apache/zookeeper/server/quorum/QuorumCnxManager.java index 57cf8d91575..344562fd92f 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/QuorumCnxManager.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/QuorumCnxManager.java @@ -31,14 +31,17 @@ import java.nio.channels.UnresolvedAddressException; import java.util.Enumeration; import java.util.Map; +import java.util.NoSuchElementException; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import java.util.NoSuchElementException; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.zookeeper.SSLCertCfg; +import org.apache.zookeeper.common.X509Exception; import org.apache.zookeeper.server.ZooKeeperThread; import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; +import org.apache.zookeeper.server.quorum.util.QuorumSocketFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -229,7 +232,7 @@ public QuorumCnxManager(QuorumPeer self) { this.self = self; // Starts listener thread that waits for connection requests - listener = new Listener(); + listener = new Listener(this.self.socketFactory); listener.setName("QuorumPeerListener"); } @@ -239,8 +242,12 @@ public QuorumCnxManager(QuorumPeer self) { * @param sid */ public void testInitiateConnection(long sid) throws Exception { - LOG.debug("Opening channel to server " + sid); - Socket sock = new Socket(); + if (LOG.isDebugEnabled()) { + LOG.debug("Opening channel to server " + sid); + } + Socket sock = this.self.socketFactory.buildForClient( + self.getVotingView().get(sid).electionAddr, + self.getVotingView().get(sid).getSslCertCfg()); setSockOpts(sock); sock.connect(self.getVotingView().get(sid).electionAddr, cnxTO); initiateConnection(sock, sid); @@ -366,7 +373,7 @@ public void receiveConnection(Socket sock) { closeSocket(sock); if (electionAddr != null) { - connectOne(sid, electionAddr); + connectOne(sid, electionAddr, getSidCertFingerPrint(sid)); } else { connectOne(sid); } @@ -429,7 +436,9 @@ public void toSend(Long sid, ByteBuffer b) { * @param sid server id * @return boolean success indication */ - synchronized private boolean connectOne(long sid, InetSocketAddress electionAddr){ + synchronized private boolean connectOne(long sid, + InetSocketAddress electionAddr, + final SSLCertCfg sslCertCfg) { if (senderWorkerMap.get(sid) != null) { LOG.debug("There is a connection already for server " + sid); return true; @@ -438,7 +447,8 @@ synchronized private boolean connectOne(long sid, InetSocketAddress electionAddr Socket sock = null; try { LOG.debug("Opening channel to server " + sid); - sock = new Socket(); + sock = this.self.socketFactory.buildForClient( + electionAddr, sslCertCfg); setSockOpts(sock); sock.connect(electionAddr, cnxTO); LOG.debug("Connected to server " + sid); @@ -453,7 +463,7 @@ synchronized private boolean connectOne(long sid, InetSocketAddress electionAddr + " at election address " + electionAddr, e); closeSocket(sock); throw e; - } catch (IOException e) { + } catch (IOException | X509Exception e) { LOG.warn("Cannot open channel to " + sid + " at election address " + electionAddr, e); @@ -483,14 +493,16 @@ synchronized void connectOne(long sid){ Map lastProposedView = lastSeenQV.getAllMembers(); if (lastCommittedView.containsKey(sid)) { knownId = true; - if (connectOne(sid, lastCommittedView.get(sid).electionAddr)) + if (connectOne(sid, lastCommittedView.get(sid).electionAddr, + lastCommittedView.get(sid).getSslCertCfg())) return; } if (lastSeenQV != null && lastProposedView.containsKey(sid) && (!knownId || (lastProposedView.get(sid).electionAddr != lastCommittedView.get(sid).electionAddr))) { knownId = true; - if (connectOne(sid, lastProposedView.get(sid).electionAddr)) + if (connectOne(sid, lastProposedView.get(sid).electionAddr, + lastCommittedView.get(sid).getSslCertCfg())) return; } if (!knownId) { @@ -499,7 +511,28 @@ synchronized void connectOne(long sid){ } } } - + + /** + * + * @param sid of the server + * @return SSLCertCfg. + */ + private SSLCertCfg getSidCertFingerPrint(final long sid) { + synchronized (self) { + if (self.getView().containsKey(sid)) { + return self.getView().get(sid).getSslCertCfg(); + } else if (self.getLastSeenQuorumVerifier() != null && + self.getLastSeenQuorumVerifier().getAllMembers() + .containsKey(sid) + && (self.getLastSeenQuorumVerifier().getAllMembers() + .get(sid).electionAddr != + self.getView().get(sid).electionAddr)) { + return self.getLastSeenQuorumVerifier() + .getAllMembers().get(sid).getSslCertCfg(); + } + } + return null; + } /** * Try to establish a connection with each server if one @@ -604,12 +637,14 @@ public QuorumPeer getQuorumPeer() { */ public class Listener extends ZooKeeperThread { + private final QuorumSocketFactory socketFactory; volatile ServerSocket ss = null; - public Listener() { + public Listener(final QuorumSocketFactory socketFactory) { // During startup of thread, thread name will be overridden to // specific election address super("ListenerThread"); + this.socketFactory = socketFactory; } /** @@ -622,8 +657,6 @@ public void run() { Socket client = null; while((!shutdown) && (numRetries < 3)){ try { - ss = new ServerSocket(); - ss.setReuseAddress(true); if (self.getQuorumListenOnAllIPs()) { int port = self.getElectionAddress().getPort(); addr = new InetSocketAddress(port); @@ -635,7 +668,9 @@ public void run() { } LOG.info("My election bind port: " + addr.toString()); setName(addr.toString()); - ss.bind(addr); + ss = this.socketFactory.buildForServer( + self, addr.getPort(), addr.getAddress()); + ss.setReuseAddress(true); while (!shutdown) { client = ss.accept(); setSockOpts(client); @@ -644,7 +679,7 @@ public void run() { receiveConnection(client); numRetries = 0; } - } catch (IOException e) { + } catch (IOException | X509Exception e) { if (shutdown) { break; } diff --git a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java index 38b02993739..2a239a8787f 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java @@ -32,6 +32,10 @@ import java.net.SocketException; import java.net.UnknownHostException; import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateEncodingException; +import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -44,9 +48,11 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.zookeeper.KeeperException.BadArgumentsException; +import org.apache.zookeeper.SSLCertCfg; import org.apache.zookeeper.common.AtomicFileWritingIdiom; import org.apache.zookeeper.common.AtomicFileWritingIdiom.WriterStatement; import org.apache.zookeeper.common.Time; +import org.apache.zookeeper.common.X509Util; import org.apache.zookeeper.jmx.MBeanRegistry; import org.apache.zookeeper.jmx.ZKMBeanInfo; import org.apache.zookeeper.server.ServerCnxnFactory; @@ -60,6 +66,7 @@ import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException; import org.apache.zookeeper.server.quorum.flexible.QuorumMaj; import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; +import org.apache.zookeeper.server.quorum.util.QuorumSocketFactory; import org.apache.zookeeper.server.util.ZxidUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -109,6 +116,8 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider */ private ZKDatabase zkDb; + + public static class QuorumServer { public InetSocketAddress addr = null; @@ -117,8 +126,10 @@ public static class QuorumServer { public InetSocketAddress clientAddr = null; public long id; - + public LearnerType type = LearnerType.PARTICIPANT; + + private SSLCertCfg sslCertCfg; private List myAddrs; @@ -166,6 +177,19 @@ public void recreateSocketAddresses() { this.electionAddr = new InetSocketAddress(address, port); } + private void setType(final HashMap propKvMap) + throws ConfigException { + if (propKvMap.containsKey("participant") && + propKvMap.containsKey("observer")) { + throw new ConfigException("Contains both participant and " + + "observer type"); + } else if (propKvMap.containsKey("participant")) { + type = LearnerType.PARTICIPANT; + } else if (propKvMap.containsKey("observer")) { + type = LearnerType.OBSERVER; + } + } + private void setType(String s) throws ConfigException { if (s.toLowerCase().equals("observer")) { type = LearnerType.OBSERVER; @@ -176,6 +200,10 @@ private void setType(String s) throws ConfigException { } } + public SSLCertCfg getSslCertCfg() { + return sslCertCfg; + } + private static String[] splitWithLeadingHostname(String s) throws ConfigException { @@ -206,7 +234,7 @@ public QuorumServer(long sid, String addressStr) throws ConfigException { String serverClientParts[] = addressStr.split(";"); String serverParts[] = splitWithLeadingHostname(serverClientParts[0]); if ((serverClientParts.length > 2) || (serverParts.length < 3) - || (serverParts.length > 4)) { + || (serverParts.length > 6)) { throw new ConfigException(addressStr + wrongFormat); } @@ -242,8 +270,23 @@ public QuorumServer(long sid, String addressStr) throws ConfigException { throw new ConfigException("Address unresolved: " + serverParts[0] + ":" + serverParts[2]); } + // Now we can expect the following + // participant:cert::cacert: + this.sslCertCfg = new SSLCertCfg(); if (serverParts.length == 4) { + // backward compatibility first. setType(serverParts[3]); + } else if (serverParts.length >= 4) { + // Parse this: cert:SHA-256-XXXX or cacert:SHA-256-XXX + // cert is self signed, cacert's fingerprint is optional. + final HashMap propKvMap = new HashMap<>(); + for (int i = 3; i < serverParts.length; i++) { + propKvMap.put(serverParts[i].trim().toLowerCase(), i); + } + + setType(propKvMap); + this.sslCertCfg = + SSLCertCfg.parseCertCfgStr(serverClientParts[0]); } setMyAddrs(); @@ -296,7 +339,16 @@ public String toString(){ sw.append(String.valueOf(electionAddr.getPort())); } if (type == LearnerType.OBSERVER) sw.append(":observer"); - else if (type == LearnerType.PARTICIPANT) sw.append(":participant"); + else if (type == LearnerType.PARTICIPANT) sw.append(":participant"); + if (isSelfSigned()) { + sw.append(":cert:"); + sw.append(sslCertCfg.getCertFingerPrintStr()); + } else if (isCASigned()) { + sw.append(":cacert:"); + if (sslCertCfg.getCertFingerPrintStr() != null) { + sw.append(sslCertCfg.getCertFingerPrintStr()); + } + } if (clientAddr!=null){ sw.append(";"); sw.append(delimitedHostString(clientAddr)); @@ -306,6 +358,14 @@ public String toString(){ return sw.toString(); } + public boolean isSelfSigned() { + return sslCertCfg != null && sslCertCfg.isSelfSigned(); + } + + public boolean isCASigned() { + return sslCertCfg != null && sslCertCfg.isCASigned(); + } + public int hashCode() { assert false : "hashCode not designed"; return 42; // any arbitrary constant will do @@ -711,6 +771,7 @@ public void setClientAddress(InetSocketAddress addr){ ServerCnxnFactory cnxnFactory; ServerCnxnFactory secureCnxnFactory; + QuorumSocketFactory socketFactory; private FileTxnSnapLog logFactory = null; @@ -733,9 +794,10 @@ public QuorumPeer() { public QuorumPeer(Map quorumPeers, File dataDir, File dataLogDir, int electionType, long myid, int tickTime, int initLimit, int syncLimit, - ServerCnxnFactory cnxnFactory) throws IOException { + ServerCnxnFactory cnxnFactory, + QuorumSocketFactory socketFactory) throws IOException { this(quorumPeers, dataDir, dataLogDir, electionType, myid, tickTime, - initLimit, syncLimit, false, cnxnFactory, + initLimit, syncLimit, false, cnxnFactory, socketFactory, new QuorumMaj(quorumPeers)); } @@ -744,9 +806,11 @@ public QuorumPeer(Map quorumPeers, File dataDir, long myid, int tickTime, int initLimit, int syncLimit, boolean quorumListenOnAllIPs, ServerCnxnFactory cnxnFactory, + QuorumSocketFactory socketFactory, QuorumVerifier quorumConfig) throws IOException { this(); this.cnxnFactory = cnxnFactory; + this.socketFactory = socketFactory; this.electionType = electionType; this.myid = myid; this.tickTime = tickTime; @@ -883,6 +947,7 @@ public QuorumPeer(Map quorumPeers, File snapDir, { this(quorumPeers, snapDir, logDir, electionAlg, myid, tickTime, initLimit, syncLimit, false, ServerCnxnFactory.createFactory(getClientAddress(quorumPeers, myid, clientPort), -1), + QuorumSocketFactory.createWithoutSSL(), new QuorumMaj(quorumPeers)); } @@ -899,6 +964,7 @@ public QuorumPeer(Map quorumPeers, File snapDir, this(quorumPeers, snapDir, logDir, electionAlg, myid,tickTime, initLimit,syncLimit, false, ServerCnxnFactory.createFactory(getClientAddress(quorumPeers, myid, clientPort), -1), + QuorumSocketFactory.createWithoutSSL(), quorumConfig); } @@ -1526,6 +1592,91 @@ public QuorumVerifier setQuorumVerifier(QuorumVerifier qv, boolean writeToDisk){ } } + public String getQuorumServerFingerPrintByElectionAddress( + final InetAddress serverElectionAddr) { + synchronized(this) { + String serverFp = + getQuorumServerFingerPrintByElectionAddress(getView(), + serverElectionAddr); + + if (serverFp == null && getLastSeenQuorumVerifier() != null) { + serverFp = + getQuorumServerFingerPrintByElectionAddress( + getLastSeenQuorumVerifier().getAllMembers(), + serverElectionAddr); + + } + + return serverFp; + } + } + + public String getQuorumServerFingerPrintByCert( + final X509Certificate cert) throws CertificateEncodingException, + NoSuchAlgorithmException { + synchronized(this) { + String serverFp = getQuorumServerFingerPrintByCert(getView(), cert); + + if (serverFp == null && getLastSeenQuorumVerifier() != null) { + serverFp = getQuorumServerFingerPrintByCert( + getLastSeenQuorumVerifier().getAllMembers(), cert); + + } + + return serverFp; + } + } + /** + * Iterate through the map and return the fingerprint for the matching + * election address. Return null if could not find it. + * @param quorumServerMap + * @return String Fingerprint , null if could not find. + */ + private String getQuorumServerFingerPrintByElectionAddress( + final Map quorumServerMap, + final InetAddress serverElectionAddr) { + for (QuorumServer quorumServer : quorumServerMap.values()) { + if (quorumServer.electionAddr.getAddress() + .equals(serverElectionAddr)) { + return quorumServer.getSslCertCfg().getCertFingerPrintStr(); + } + } + + return null; + } + + private String getQuorumServerFingerPrintByCert( + final Map quorumServerMap, + final X509Certificate incomingPeerCert) + throws CertificateEncodingException, NoSuchAlgorithmException { + for (QuorumServer quorumServer : quorumServerMap.values()) { + final String peerFp = + quorumServer.getSslCertCfg().getCertFingerPrintStr(); + final MessageDigest peerFpSupportedMd = + X509Util.getSupportedMessageDigestForFpStr(peerFp); + if (peerFpSupportedMd == null) { + final String errStr = + "QuorumServer: " + quorumServer + + " un-supported finger print in dynamic cfg: " + + peerFp; + LOG.error(errStr); + throw new IllegalAccessError(errStr); + } + + LOG.info("quorumServer: " + quorumServer + " cert:" + peerFp + + " , given cert:" + + SSLCertCfg.getDigestToCertFp( + X509Util.getMessageDigestFromCert(incomingPeerCert, + peerFpSupportedMd.getAlgorithm()))); + if (X509Util.validateCert(peerFpSupportedMd, peerFp, + incomingPeerCert)) { + return peerFp; + } + } + + return null; + } + private String makeDynamicConfigFilename(long version) { return configFilename + ".dynamic." + Long.toHexString(version); } @@ -1613,6 +1764,13 @@ public void setCnxnFactory(ServerCnxnFactory cnxnFactory) { public void setSecureCnxnFactory(ServerCnxnFactory secureCnxnFactory) { this.secureCnxnFactory = secureCnxnFactory; + if (secureCnxnFactory != null) { + secureCnxnFactory.setQuorumPeer(this); + } + } + + public void setSocketFactory(QuorumSocketFactory socketFactory) { + this.socketFactory = socketFactory; } private void startServerCnxnFactory() { diff --git a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerMain.java b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerMain.java index cde193e8685..25c5e1f8ade 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerMain.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerMain.java @@ -18,20 +18,23 @@ package org.apache.zookeeper.server.quorum; import java.io.IOException; +import java.security.NoSuchAlgorithmException; import javax.management.JMException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.zookeeper.common.X509Exception; import org.apache.zookeeper.jmx.ManagedUtil; +import org.apache.zookeeper.server.DatadirCleanupManager; import org.apache.zookeeper.server.ServerCnxnFactory; import org.apache.zookeeper.server.ZKDatabase; -import org.apache.zookeeper.server.DatadirCleanupManager; import org.apache.zookeeper.server.ZooKeeperServerMain; import org.apache.zookeeper.server.admin.AdminServer.AdminServerException; import org.apache.zookeeper.server.persistence.FileTxnSnapLog; import org.apache.zookeeper.server.persistence.FileTxnSnapLog.DatadirException; import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException; +import org.apache.zookeeper.server.quorum.util.QuorumSocketFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @@ -117,7 +120,13 @@ protected void initializeAndRun(String[] args) purgeMgr.start(); if (args.length == 1 && config.isDistributed()) { - runFromConfig(config); + try { + runFromConfig(config); + } catch (NoSuchAlgorithmException + | X509Exception.KeyManagerException + | X509Exception.TrustManagerException exp) { + throw new ConfigException("SSL init error", exp); + } } else { LOG.warn("Either no config or no quorum defined in config, running " + " in standalone mode"); @@ -126,9 +135,10 @@ protected void initializeAndRun(String[] args) } } - public void runFromConfig(QuorumPeerConfig config) - throws IOException, AdminServerException - { + public void runFromConfig(QuorumPeerConfig config) throws IOException, + AdminServerException, NoSuchAlgorithmException, + X509Exception.KeyManagerException, + X509Exception.TrustManagerException { try { ManagedUtil.registerLog4jMBeans(); } catch (JMException e) { @@ -154,7 +164,11 @@ public void runFromConfig(QuorumPeerConfig config) true); } - quorumPeer = getQuorumPeer(); + QuorumSocketFactory socketFactory = + QuorumSocketFactory.createDefault(); + + quorumPeer = new QuorumPeer(); + quorumPeer.setTxnFactory(new FileTxnSnapLog( config.getDataLogDir(), config.getDataDir())); @@ -178,6 +192,7 @@ public void runFromConfig(QuorumPeerConfig config) quorumPeer.initConfigInZKDatabase(); quorumPeer.setCnxnFactory(cnxnFactory); quorumPeer.setSecureCnxnFactory(secureCnxnFactory); + quorumPeer.setSocketFactory(socketFactory); quorumPeer.setLearnerType(config.getPeerType()); quorumPeer.setSyncEnabled(config.getSyncEnabled()); quorumPeer.setQuorumListenOnAllIPs(config.getQuorumListenOnAllIPs()); diff --git a/src/java/main/org/apache/zookeeper/server/quorum/util/CertificateVerificationException.java b/src/java/main/org/apache/zookeeper/server/quorum/util/CertificateVerificationException.java new file mode 100644 index 00000000000..2cbb7a8f79e --- /dev/null +++ b/src/java/main/org/apache/zookeeper/server/quorum/util/CertificateVerificationException.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.server.quorum.util; + +/** + * This class wraps an exception that could be thrown during + * the certificate verification process. + * + * @author Svetlin Nakov + */ +public class CertificateVerificationException extends Exception { + private static final long serialVersionUID = 1L; + + public CertificateVerificationException(String message, Throwable cause) { + super(message, cause); + } + + public CertificateVerificationException(String message) { + super(message); + } +} \ No newline at end of file diff --git a/src/java/main/org/apache/zookeeper/server/quorum/util/CertificateVerifier.java b/src/java/main/org/apache/zookeeper/server/quorum/util/CertificateVerifier.java new file mode 100644 index 00000000000..837d67874ac --- /dev/null +++ b/src/java/main/org/apache/zookeeper/server/quorum/util/CertificateVerifier.java @@ -0,0 +1,182 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.server.quorum.util; + +import java.security.GeneralSecurityException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.PublicKey; +import java.security.SignatureException; +import java.security.cert.CertPathBuilder; +import java.security.cert.CertPathBuilderException; +import java.security.cert.CertStore; +import java.security.cert.CertificateException; +import java.security.cert.CollectionCertStoreParameters; +import java.security.cert.PKIXBuilderParameters; +import java.security.cert.PKIXCertPathBuilderResult; +import java.security.cert.TrustAnchor; +import java.security.cert.X509CertSelector; +import java.security.cert.X509Certificate; +import java.util.HashSet; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * XXX: From - http://www.nakov.com/blog/2009/12/01/x509-certificate-validation-in-java-build-and-verify-chain-and-verify-clr-with-bouncy-castle/ + * Class for building a certification chain for given certificate and verifying + * it. Relies on a set of root CA certificates and intermediate certificates + * that will be used for building the certification chain. The verification + * process assumes that all self-signed certificates in the set are trusted + * root CA certificates and all other certificates in the set are intermediate + * certificates. + * + * @author Svetlin Nakov + */ +public class CertificateVerifier { + private static final Logger LOG + = LoggerFactory.getLogger(CertificateVerifier.class); + /** + * Attempts to build a certification chain for given certificate and to + * verify it. Relies on a set of root CA certificates and intermediate + * certificates that will be used for building the certification chain. + * The verification process assumes that all self-signed certificates in + * the set are trusted root CA certificates and all other certificates in + * the set are intermediate + * certificates. + * + * @param cert - certificate for validation + * @param additionalCerts - set of trusted root CA certificates that will + * be used as "trust anchors" and intermediate CA + * certificates that will be used as part of the + * certification chain. All self-signed + * certificates are considered to be trusted root CA + * certificates. All the rest are considered to be + * intermediate CA certificates. + * @return the certification chain (if verification is successful) + * @throws CertificateVerificationException - if the certification is not + * successful (e.g. certification path cannot be built or some + * certificate in the chain is expired or CRL checks are failed) + */ + public static PKIXCertPathBuilderResult verifyCertificate( + X509Certificate cert, Set additionalCerts) + throws CertificateVerificationException { + try { + // Check for self-signed certificate + if (isSelfSigned(cert)) { + throw new CertificateVerificationException( + "The certificate is self-signed."); + } + + // Prepare a set of trusted root CA certificates + // and a set of intermediate certificates + Set trustedRootCerts = new HashSet<>(); + Set intermediateCerts = new HashSet<>(); + for (X509Certificate additionalCert : additionalCerts) { + if (isSelfSigned(additionalCert)) { + trustedRootCerts.add(additionalCert); + } else { + intermediateCerts.add(additionalCert); + } + } + + // Attempt to build the certification chain and verify it + PKIXCertPathBuilderResult verifiedCertChain = + verifyCertificate(cert, trustedRootCerts, + intermediateCerts); + + // The chain is built and verified. Return it as a result + return verifiedCertChain; + } catch (CertPathBuilderException certPathEx) { + throw new CertificateVerificationException( + "Error building certification path: " + + cert.getSubjectX500Principal(), certPathEx); + } catch (CertificateVerificationException cvex) { + throw cvex; + } catch (Exception ex) { + throw new CertificateVerificationException( + "Error verifying the certificate: " + + cert.getSubjectX500Principal(), ex); + } + } + + /** + * Checks whether given X.509 certificate is self-signed. + */ + public static boolean isSelfSigned(X509Certificate cert) { + try { + // Try to verify certificate signature with its own public key + PublicKey key = cert.getPublicKey(); + cert.verify(key); + return true; + } catch (SignatureException | InvalidKeyException | + NoSuchProviderException | NoSuchAlgorithmException + | CertificateException exp) { + // not self signed? + return false; + } + } + + /** + * Attempts to build a certification chain for given certificate and to + * verify it. Relies on a set of root CA certificates (trust anchors) and + * a set of intermediate certificates (to be used as part of the chain). + * @param cert - certificate for validation + * @param trustedRootCerts - set of trusted root CA certificates + * @param intermediateCerts - set of intermediate certificates + * @return the certification chain (if verification is successful) + * @throws GeneralSecurityException - if the verification is not successful + * (e.g. certification path cannot be built or some certificate in the + * chain is expired) + */ + private static PKIXCertPathBuilderResult verifyCertificate( + X509Certificate cert, Set trustedRootCerts, + Set intermediateCerts) + throws GeneralSecurityException { + + // Create the selector that specifies the starting certificate + X509CertSelector selector = new X509CertSelector(); + selector.setCertificate(cert); + + // Create the trust anchors (set of root CA certificates) + Set trustAnchors = new HashSet<>(); + for (X509Certificate trustedRootCert : trustedRootCerts) { + trustAnchors.add(new TrustAnchor(trustedRootCert, null)); + } + + // Configure the PKIX certificate builder algorithm parameters + PKIXBuilderParameters pkixParams = + new PKIXBuilderParameters(trustAnchors, selector); + + // Disable CRL checks (this is done manually as additional step) + pkixParams.setRevocationEnabled(false); + + // Specify a list of intermediate certificates + CertStore intermediateCertStore = CertStore.getInstance("Collection", + new CollectionCertStoreParameters(intermediateCerts)); + pkixParams.addCertStore(intermediateCertStore); + + // Build and verify the certification chain + CertPathBuilder builder = CertPathBuilder.getInstance("PKIX"); + PKIXCertPathBuilderResult result = + (PKIXCertPathBuilderResult) builder.build(pkixParams); + return result; + } +} diff --git a/src/java/main/org/apache/zookeeper/server/quorum/util/QuorumSocketFactory.java b/src/java/main/org/apache/zookeeper/server/quorum/util/QuorumSocketFactory.java new file mode 100644 index 00000000000..e8800c6f885 --- /dev/null +++ b/src/java/main/org/apache/zookeeper/server/quorum/util/QuorumSocketFactory.java @@ -0,0 +1,178 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.server.quorum.util; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.security.NoSuchAlgorithmException; + +import javax.net.ssl.SSLServerSocket; + +import org.apache.zookeeper.SSLCertCfg; +import org.apache.zookeeper.common.X509Exception; +import org.apache.zookeeper.common.X509Util; +import org.apache.zookeeper.common.ZKConfig; +import org.apache.zookeeper.server.quorum.QuorumPeer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Helps with abstracting away SSL gory details for consumers. + * details. + */ +public class QuorumSocketFactory { + private static final Logger LOG = + LoggerFactory.getLogger(QuorumSocketFactory.class.getName()); + public static final String SSL_ENABLED_PROP = "quorum.ssl.enabled"; + private static final int LISTEN_BACKLOG = 20; + private final boolean sslEnabled; + + private QuorumSocketFactory(boolean sslEnabled) { + this.sslEnabled = sslEnabled; + } + + public static QuorumSocketFactory createDefault() + throws NoSuchAlgorithmException, X509Exception.KeyManagerException, + X509Exception.TrustManagerException { + String propValue = System.getProperty( + QuorumSocketFactory.SSL_ENABLED_PROP); + if (propValue != null && + propValue.compareToIgnoreCase("true") == 0) { + return QuorumSocketFactory.createForSSL(); + } else { + return QuorumSocketFactory.createWithoutSSL(); + } + } + + public static QuorumSocketFactory createWithoutSSL() { + return new QuorumSocketFactory(false); + } + + public static QuorumSocketFactory createForSSL() + throws NoSuchAlgorithmException, X509Exception.KeyManagerException, + X509Exception.TrustManagerException { + return new QuorumSocketFactory(true); + } + + public ServerSocket buildForServer(final QuorumPeer quorumPeer, + final int listenPort, + final InetAddress bindAddr) + throws X509Exception, IOException { + return buildForServer(quorumPeer, listenPort, LISTEN_BACKLOG, bindAddr); + } + + public ServerSocket buildForServer(final QuorumPeer quorumPeer, + final int port) + throws X509Exception, IOException { + return buildForServer(quorumPeer, port, LISTEN_BACKLOG, null); + } + + public ServerSocket buildForServer(final QuorumPeer quorumPeer, + final int listenPort, + final int backlog, + final InetAddress bindAddr) + throws X509Exception, IOException { + ServerSocket s = null; + if (this.sslEnabled) { + s = newSslServerSocket(quorumPeer, listenPort, backlog, bindAddr); + } else { + s = newServerSocket(listenPort, backlog, bindAddr); + } + s.setReuseAddress(true); + return s; + } + + public Socket buildForClient(final InetSocketAddress peerAddr, + final SSLCertCfg sslCertCfg) throws + X509Exception, + IOException { + if (this.sslEnabled) { + return newSslSocket(peerAddr, sslCertCfg); + } else { + return newSocket(); + } + } + + private Socket newSocket() throws IOException { + return new Socket(); + } + + private Socket newSslSocket(final InetSocketAddress peerAddr, + final SSLCertCfg sslCertCfg) + throws X509Exception, IOException { + Socket clientSocket = null; + try { + clientSocket = X509Util.createSSLContext( + new ZKConfig(), peerAddr, sslCertCfg) + .getSocketFactory() + .createSocket(); + } catch (X509Exception.SSLContextException exp) { + LOG.error("failed creating ssl client socket, exp: " + exp); + throw new X509Exception(exp); + } catch (IOException exp) { + LOG.error("failed creating ssl client socket, exp: " + exp); + throw exp; + } + + return clientSocket; + } + + private ServerSocket newServerSocket(final int port, final int backlog, + final InetAddress listenAddr) + throws IOException { + if (listenAddr != null) { + return new ServerSocket(port, backlog, listenAddr); + } else { + return new ServerSocket(port, backlog); + } + } + + private ServerSocket newSslServerSocket( + final QuorumPeer quorumPeer, final int port, + final int backlog, final InetAddress listenAddr) + throws X509Exception { + SSLServerSocket serverSocket = null; + try { + if (listenAddr != null) { + serverSocket = + (SSLServerSocket)X509Util.createSSLContext( + new ZKConfig(), quorumPeer) + .getServerSocketFactory() + .createServerSocket(port, backlog, listenAddr); + } else { + // bind to any address + serverSocket = + (SSLServerSocket)X509Util.createSSLContext( + new ZKConfig(), quorumPeer) + .getServerSocketFactory() + .createServerSocket(port, backlog); + } + } catch (X509Exception.SSLContextException | IOException exp) { + LOG.error("creating server socket, exp: " + exp); + throw new X509Exception(exp); + } + + // Fail if client does not provide credentials. + serverSocket.setNeedClientAuth(true); + return serverSocket; + } + +} diff --git a/src/java/main/org/apache/zookeeper/server/quorum/util/ZKDynamicX509TrustManager.java b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKDynamicX509TrustManager.java new file mode 100644 index 00000000000..f05c226320d --- /dev/null +++ b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKDynamicX509TrustManager.java @@ -0,0 +1,178 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.quorum.util; + +import java.net.InetAddress; +import java.net.Socket; +import java.net.UnknownHostException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; + +import javax.net.ssl.SSLEngine; +import javax.net.ssl.X509ExtendedTrustManager; + +import org.apache.commons.validator.routines.InetAddressValidator; +import org.apache.zookeeper.common.X509Util; +import org.apache.zookeeper.server.quorum.QuorumPeer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ZKDynamicX509TrustManager extends X509ExtendedTrustManager { + private static final Logger LOG + = LoggerFactory.getLogger(ZKPeerX509TrustManager.class); + + private final QuorumPeer quorumPeer; + + public ZKDynamicX509TrustManager(final QuorumPeer quorumPeer) { + this.quorumPeer = quorumPeer; + } + + @Override + public void checkClientTrusted(X509Certificate[] x509Certificates, String s) + throws CertificateException { + + } + + @Override + public void checkServerTrusted(X509Certificate[] x509Certificates, String s) + throws CertificateException { + + } + + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[]{}; + } + + public void checkClientTrusted( + final X509Certificate[] certs, final String authType, + final Socket socket) + throws CertificateException { + validateSingleAndSelfSigned(certs); + } + + public void checkServerTrusted( + final X509Certificate[] certs, final String authType, + final Socket socket) + throws CertificateException { + validateSingleAndSelfSigned(certs); + } + + @Override + public void checkClientTrusted(X509Certificate[] x509Certificates, String s, + SSLEngine sslEngine) + throws CertificateException { + + } + + @Override + public void checkServerTrusted(X509Certificate[] x509Certificates, String s, + SSLEngine sslEngine) + throws CertificateException { + + } + + private void validateSingleAndSelfSigned(final X509Certificate[] certs) + throws CertificateException { + if (certs.length == 0) { + final String errStr = "Invalid server, did not send any cert"; + LOG.error(errStr); + throw new CertificateException(errStr); + } + + try { + validatePeerCert(certs[0]); + } catch (NoSuchAlgorithmException exp) { + final String errStr = "Unable to validate peer cert"; + LOG.error("{}", errStr, exp); + throw new CertificateException(errStr, exp); + } + } + + private void validatePeerCert( + final X509Certificate cert, final String peerHost) + throws CertificateException { + if(peerHost == null || peerHost.trim().isEmpty()) { + final String errStr = "Invalid peerHost, cannot be null or empty"; + LOG.error(errStr); + throw new CertificateException(errStr); + } + + if (!InetAddressValidator.getInstance().isValid(peerHost)) { + final String errStr = "Invalid peerHost, should be a valid IP"; + LOG.error(errStr); + throw new CertificateException(errStr); + } + + // We have a valid ip address + try { + validatePeerCert(cert, InetAddress.getByName(peerHost)); + } catch (UnknownHostException exp) { + final String errStr = "Invalid peerHost, should be a valid IP"; + LOG.error("{}", errStr, exp); + throw new CertificateException(errStr, exp); + } + } + + private void validatePeerCert(final X509Certificate cert, + final InetAddress peerAddr) + throws CertificateException { + + LOG.info("validating cert: " + cert); + // Verify that server presented a self-signed cert. + X509Util.verifySelfSigned(cert); + + final String peerCertFingerPrint = + quorumPeer.getQuorumServerFingerPrintByElectionAddress( + peerAddr); + + // If we could not get the fp then bail!. + if (peerCertFingerPrint == null) { + final String errStr = "Invalid cert and peerAddr: " + + peerAddr + " could not find fingerprint for this address"; + LOG.error(errStr); + throw new CertificateException(errStr); + } + + validatePeerCert(cert, peerCertFingerPrint); + LOG.info("validation done"); + } + + private void validatePeerCert(final X509Certificate cert) + throws CertificateException, NoSuchAlgorithmException { + if (quorumPeer == null) { + throw new IllegalAccessError("Cannot be used this way, quorumPeer" + + " is null"); + } + + LOG.info("validating cert: " + cert); + final String peerCertFingerPrint = + quorumPeer.getQuorumServerFingerPrintByCert(cert); + + // If we could not get the fp then bail!. + if (peerCertFingerPrint == null) { + final String errStr = "Invalid cert"; + LOG.error(errStr); + throw new CertificateException(errStr); + } + + LOG.info("validation done"); + } +} + diff --git a/src/java/main/org/apache/zookeeper/server/quorum/util/ZKPeerX509TrustManager.java b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKPeerX509TrustManager.java new file mode 100644 index 00000000000..d08da78c029 --- /dev/null +++ b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKPeerX509TrustManager.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.server.quorum.util; + +import java.net.InetSocketAddress; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; + +import javax.net.ssl.X509TrustManager; + +import org.apache.zookeeper.common.X509Util; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ZKPeerX509TrustManager implements X509TrustManager { + private static final Logger LOG + = LoggerFactory.getLogger(ZKPeerX509TrustManager.class); + + private final InetSocketAddress peerAddr; // TODO: Host verification? + private final String peerCertFingerPrintStr; + + public ZKPeerX509TrustManager( + final InetSocketAddress peerAddr, + final String peerCertFingerPrintStr) { + this.peerAddr = peerAddr; + this.peerCertFingerPrintStr = peerCertFingerPrintStr; + } + + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[]{}; + } + + public void checkClientTrusted( + final X509Certificate[] certs, final String authType) + throws CertificateException { + LOG.error("Not supported!"); + throw new IllegalAccessError("Not Implemented!"); + } + + public void checkServerTrusted( + final X509Certificate[] certs, final String authType) + throws CertificateException { + if (certs.length == 0) { + final String errStr = "Invalid server, did not send any cert"; + LOG.error(errStr); + throw new CertificateException(errStr); + } + + validatePeerCert(certs[0]); + } + + private void validatePeerCert(final X509Certificate cert) + throws CertificateException { + // Verify that server presented a self-signed cert. + X509Util.verifySelfSigned(cert); + + try { + X509Util.validateCert(peerCertFingerPrintStr, cert); + } catch (NoSuchAlgorithmException exp) { + throw new CertificateException(exp); + } + } +} diff --git a/src/java/main/org/apache/zookeeper/server/quorum/util/ZKX509TrustManager.java b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKX509TrustManager.java new file mode 100644 index 00000000000..4a5fe023e65 --- /dev/null +++ b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKX509TrustManager.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.server.quorum.util; + +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.SignatureException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.HashSet; +import java.util.Set; + +import javax.net.ssl.X509TrustManager; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ZKX509TrustManager implements X509TrustManager { + private static final Logger LOG + = LoggerFactory.getLogger(ZKX509TrustManager.class); + private final X509Certificate rootCACert; + + public ZKX509TrustManager(final X509Certificate rootCACert) { + this.rootCACert = rootCACert; + } + + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[] { this.rootCACert }; + } + + public void checkClientTrusted(X509Certificate[] certs, + String authType) throws CertificateException { + try { + validateCertChain(certs); + } catch (CertificateVerificationException exp) { + throw new CertificateException(exp); + } + } + + public void checkServerTrusted(X509Certificate[] certs, + String authType) throws CertificateException { + try { + validateCertChain(certs); + } catch (CertificateVerificationException exp) { + throw new CertificateException(exp); + } + } + + private void validateCertChain(X509Certificate[] certs) + throws CertificateVerificationException { + X509Certificate clientCert = null; + Set restOfCerts = new HashSet<>(); + for (final X509Certificate cert: certs) { + // skip self signed cert + if (CertificateVerifier.isSelfSigned(cert)) { + continue; + } + + // first non self signed cert will be client cert? + // TODO: how do we get the first cert?. + if (clientCert == null) { + clientCert = cert; + } else { + restOfCerts.add(cert); + } + } + + restOfCerts.add(rootCACert); + CertificateVerifier.verifyCertificate(clientCert, restOfCerts); + } + + private void checkAllCerts(X509Certificate[] certs) + throws CertificateException { + for (X509Certificate cert : certs) { + try { + cert.verify(rootCACert.getPublicKey()); + } catch (NoSuchAlgorithmException + | InvalidKeyException | NoSuchProviderException + | SignatureException exp) { + LOG.error("cert validation failed, exp: " + exp); + throw new CertificateException(exp); + } + } + } +} diff --git a/src/java/test/org/apache/zookeeper/ClientReconnectTest.java b/src/java/test/org/apache/zookeeper/ClientReconnectTest.java index 566b915c1dd..24e6e5c5604 100644 --- a/src/java/test/org/apache/zookeeper/ClientReconnectTest.java +++ b/src/java/test/org/apache/zookeeper/ClientReconnectTest.java @@ -59,8 +59,9 @@ SocketChannel createSock() { public void testClientReconnect() throws IOException, InterruptedException { HostProvider hostProvider = mock(HostProvider.class); when(hostProvider.size()).thenReturn(1); - InetSocketAddress inaddr = new InetSocketAddress("127.0.0.1", 1111); - when(hostProvider.next(anyLong())).thenReturn(inaddr); + final ServerCfg serverCfg = new ServerCfg("127.0.0.1:1111", + new InetSocketAddress("127.0.0.1", 1111)); + when(hostProvider.next(anyLong())).thenReturn(serverCfg); ZooKeeper zk = mock(ZooKeeper.class); when(zk.getClientConfig()).thenReturn(new ZKClientConfig()); sc = SocketChannel.open(); diff --git a/src/java/test/org/apache/zookeeper/CustomHostProviderTest.java b/src/java/test/org/apache/zookeeper/CustomHostProviderTest.java index f9762d24996..0583c3034c6 100644 --- a/src/java/test/org/apache/zookeeper/CustomHostProviderTest.java +++ b/src/java/test/org/apache/zookeeper/CustomHostProviderTest.java @@ -37,15 +37,16 @@ public int size() { return 1; } @Override - public InetSocketAddress next(long spinDelay) { - return new InetSocketAddress("127.0.0.1", 2181); + public ServerCfg next(long spinDelay) { + return new ServerCfg("127.0.0.1:2181", + new InetSocketAddress("127.0.0 .1", 2181)); } @Override public void onConnected() { } @Override - public boolean updateServerList(Collection - serverAddresses, InetSocketAddress currentHost) { + public boolean updateServerList(final Collection serversCfg, + final InetSocketAddress currentHost) { counter.decrementAndGet(); return false; } diff --git a/src/java/test/org/apache/zookeeper/server/quorum/LearnerTest.java b/src/java/test/org/apache/zookeeper/server/quorum/LearnerTest.java index 85284f6d42d..93e88b7b61f 100644 --- a/src/java/test/org/apache/zookeeper/server/quorum/LearnerTest.java +++ b/src/java/test/org/apache/zookeeper/server/quorum/LearnerTest.java @@ -30,6 +30,7 @@ import org.apache.jute.BinaryInputArchive; import org.apache.jute.BinaryOutputArchive; +import org.apache.zookeeper.ServerCfg; import org.apache.zookeeper.ZKTestCase; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.data.ACL; @@ -107,10 +108,10 @@ public void connectionRetryTimeoutTest() throws Exception { learner.self.setSyncLimit(2); // this addr won't even be used since we fake the Socket.connect - InetSocketAddress addr = new InetSocketAddress(1111); + ServerCfg serverCfg = new ServerCfg("any", new InetSocketAddress(1111)); // we expect this to throw an IOException since we're faking socket connect errors every time - learner.connectToLeader(addr); + learner.connectToLeader(serverCfg); } @Test public void connectionInitLimitTimeoutTest() throws Exception { @@ -121,7 +122,7 @@ public void connectionInitLimitTimeoutTest() throws Exception { learner.self.setSyncLimit(2); // this addr won't even be used since we fake the Socket.connect - InetSocketAddress addr = new InetSocketAddress(1111); + ServerCfg serverCfg = new ServerCfg("any", new InetSocketAddress(1111)); // pretend each connect attempt takes 4000 milliseconds learner.setTimeMultiplier((long)4000 * 1000000); @@ -130,7 +131,7 @@ public void connectionInitLimitTimeoutTest() throws Exception { // we expect this to throw an IOException since we're faking socket connect errors every time try { - learner.connectToLeader(addr); + learner.connectToLeader(serverCfg); Assert.fail("should have thrown IOException!"); } catch (IOException e) { //good, wanted to see that, let's make sure we ran out of time diff --git a/src/java/test/org/apache/zookeeper/server/quorum/Zab1_0Test.java b/src/java/test/org/apache/zookeeper/server/quorum/Zab1_0Test.java index 778ea1e2989..bc81095b207 100644 --- a/src/java/test/org/apache/zookeeper/server/quorum/Zab1_0Test.java +++ b/src/java/test/org/apache/zookeeper/server/quorum/Zab1_0Test.java @@ -25,12 +25,12 @@ import java.io.BufferedReader; import java.io.ByteArrayOutputStream; +import java.io.EOFException; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; -import java.io.EOFException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.ServerSocket; @@ -45,9 +45,11 @@ import org.apache.jute.InputArchive; import org.apache.jute.OutputArchive; import org.apache.zookeeper.PortAssignment; +import org.apache.zookeeper.ServerCfg; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.Watcher.Event.EventType; +import org.apache.zookeeper.ZKTestCase; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.server.ByteBufferInputStream; @@ -68,13 +70,14 @@ import org.apache.zookeeper.txn.ErrorTxn; import org.apache.zookeeper.txn.SetDataTxn; import org.apache.zookeeper.txn.TxnHeader; -import org.apache.zookeeper.ZKTestCase; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.junit.Assert.assertEquals; + public class Zab1_0Test extends ZKTestCase { private static final int SYNC_LIMIT = 2; @@ -1275,10 +1278,10 @@ static class ConversableFollower extends Follower { public void setLeaderSocketAddress(InetSocketAddress addr) { leaderAddr = addr; } - + @Override - protected InetSocketAddress findLeader() { - return leaderAddr; + protected ServerCfg findLeader() { + return new ServerCfg("given", leaderAddr); } } private ConversableFollower createFollower(File tmpDir, QuorumPeer peer) @@ -1303,8 +1306,8 @@ public void setLeaderSocketAddress(InetSocketAddress addr) { } @Override - protected InetSocketAddress findLeader() { - return leaderAddr; + protected ServerCfg findLeader() { + return new ServerCfg("given", leaderAddr); } } diff --git a/src/java/test/org/apache/zookeeper/test/ConnectStringParserTest.java b/src/java/test/org/apache/zookeeper/test/ConnectStringParserTest.java index 393cc0363bc..b463e0f0a98 100644 --- a/src/java/test/org/apache/zookeeper/test/ConnectStringParserTest.java +++ b/src/java/test/org/apache/zookeeper/test/ConnectStringParserTest.java @@ -18,6 +18,7 @@ package org.apache.zookeeper.test; +import org.apache.zookeeper.ServerCfg; import org.apache.zookeeper.ZKTestCase; import org.apache.zookeeper.client.ConnectStringParser; import org.junit.Assert; @@ -46,8 +47,10 @@ public void testParseServersWithoutPort(){ String servers = "10.10.10.1,10.10.10.2"; ConnectStringParser parser = new ConnectStringParser(servers); - Assert.assertEquals("10.10.10.1", parser.getServerAddresses().get(0).getHostString()); - Assert.assertEquals("10.10.10.2", parser.getServerAddresses().get(1).getHostString()); + ServerCfg[] serverCfgs = parser.getServersCfg() + .toArray(new ServerCfg[parser.getServersCfg().size()]); + Assert.assertEquals("10.10.10.1", serverCfgs[0].getHostString()); + Assert.assertEquals("10.10.10.2", serverCfgs[1].getHostString()); } @Test @@ -55,11 +58,13 @@ public void testParseServersWithPort(){ String servers = "10.10.10.1:112,10.10.10.2:110"; ConnectStringParser parser = new ConnectStringParser(servers); - Assert.assertEquals("10.10.10.1", parser.getServerAddresses().get(0).getHostString()); - Assert.assertEquals("10.10.10.2", parser.getServerAddresses().get(1).getHostString()); + ServerCfg[] serverCfgs = parser.getServersCfg() + .toArray(new ServerCfg[parser.getServersCfg().size()]); + Assert.assertEquals("10.10.10.1", serverCfgs[0].getHostString()); + Assert.assertEquals("10.10.10.2", serverCfgs[1].getHostString()); - Assert.assertEquals(112, parser.getServerAddresses().get(0).getPort()); - Assert.assertEquals(110, parser.getServerAddresses().get(1).getPort()); + Assert.assertEquals(112, serverCfgs[0].getPort()); + Assert.assertEquals(110, serverCfgs[1].getPort()); } private void assertChrootPath(String expected, ConnectStringParser parser){ diff --git a/src/java/test/org/apache/zookeeper/test/LENonTerminateTest.java b/src/java/test/org/apache/zookeeper/test/LENonTerminateTest.java index 2bbf7b581aa..8678124cb82 100644 --- a/src/java/test/org/apache/zookeeper/test/LENonTerminateTest.java +++ b/src/java/test/org/apache/zookeeper/test/LENonTerminateTest.java @@ -31,8 +31,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.zookeeper.PortAssignment; import org.apache.zookeeper.ZKTestCase; import org.apache.zookeeper.server.ServerCnxnFactory; @@ -40,14 +38,17 @@ import org.apache.zookeeper.server.quorum.FLELostMessageTest; import org.apache.zookeeper.server.quorum.LeaderElection; import org.apache.zookeeper.server.quorum.QuorumPeer; -import org.apache.zookeeper.server.quorum.Vote; import org.apache.zookeeper.server.quorum.QuorumPeer.LearnerType; import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer; import org.apache.zookeeper.server.quorum.QuorumPeer.ServerState; +import org.apache.zookeeper.server.quorum.Vote; import org.apache.zookeeper.server.quorum.flexible.QuorumMaj; +import org.apache.zookeeper.server.quorum.util.QuorumSocketFactory; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @SuppressWarnings("deprecation") public class LENonTerminateTest extends ZKTestCase { @@ -219,6 +220,7 @@ public MockQuorumPeer(Map quorumPeers, File snapDir, super(quorumPeers, snapDir, logDir, electionAlg, myid,tickTime, initLimit,syncLimit, false, ServerCnxnFactory.createFactory(clientPort, -1), + QuorumSocketFactory.createWithoutSSL(), new QuorumMaj(quorumPeers)); } diff --git a/src/java/test/org/apache/zookeeper/test/StaticHostProviderTest.java b/src/java/test/org/apache/zookeeper/test/StaticHostProviderTest.java index 10c6d1c5f03..ea807821057 100644 --- a/src/java/test/org/apache/zookeeper/test/StaticHostProviderTest.java +++ b/src/java/test/org/apache/zookeeper/test/StaticHostProviderTest.java @@ -18,16 +18,6 @@ package org.apache.zookeeper.test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertTrue; - -import org.apache.zookeeper.ZKTestCase; -import org.apache.zookeeper.client.HostProvider; -import org.apache.zookeeper.client.StaticHostProvider; -import org.apache.zookeeper.common.Time; -import org.junit.Test; - import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; @@ -36,13 +26,24 @@ import java.util.HashMap; import java.util.Random; +import org.apache.zookeeper.ServerCfg; +import org.apache.zookeeper.ZKTestCase; +import org.apache.zookeeper.client.HostProvider; +import org.apache.zookeeper.client.StaticHostProvider; +import org.apache.zookeeper.common.Time; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; + public class StaticHostProviderTest extends ZKTestCase { private Random r = new Random(1); @Test public void testNextGoesRound() { HostProvider hostProvider = getHostProvider((byte) 2); - InetSocketAddress first = hostProvider.next(0); + ServerCfg first = hostProvider.next(0); assertTrue(first != null); hostProvider.next(0); assertEquals(first, hostProvider.next(0)); @@ -78,20 +79,25 @@ public void testNextDoesNotSleepForZero() { @Test(expected = IllegalArgumentException.class) public void testTwoInvalidHostAddresses() { - ArrayList list = new ArrayList(); - list.add(new InetSocketAddress("a...", 2181)); - list.add(new InetSocketAddress("b...", 2181)); + ArrayList list = new ArrayList(); + list.add(new ServerCfg("a...", new InetSocketAddress("a...", 2181))); + list.add(new ServerCfg("b...", new InetSocketAddress("b...", 2181))); new StaticHostProvider(list); } @Test public void testOneInvalidHostAddresses() { - Collection addr = getServerAddresses((byte) 1); - addr.add(new InetSocketAddress("a...", 2181)); + Collection addrs = getServerAddresses((byte) 1); - StaticHostProvider sp = new StaticHostProvider(addr); - InetSocketAddress n1 = sp.next(0); - InetSocketAddress n2 = sp.next(0); + Collection serverCfgs = new ArrayList(); + for (final InetSocketAddress addr : addrs) { + serverCfgs.add(new ServerCfg("a...", + new InetSocketAddress("a...", 2181))); + } + + StaticHostProvider sp = new StaticHostProvider(serverCfgs); + ServerCfg n1 = sp.next(0); + ServerCfg n2 = sp.next(0); assertEquals(n2, n1); } @@ -105,9 +111,9 @@ public void testTwoConsequitiveCallsToNextReturnDifferentElement() { @Test public void testOnConnectDoesNotReset() { HostProvider hostProvider = getHostProvider((byte) 2); - InetSocketAddress first = hostProvider.next(0); + ServerCfg first = hostProvider.next(0); hostProvider.onConnected(); - InetSocketAddress second = hostProvider.next(0); + ServerCfg second = hostProvider.next(0); assertNotSame(first, second); } @@ -117,7 +123,8 @@ public void testOnConnectDoesNotReset() { @Test public void testUpdateClientMigrateOrNot() throws UnknownHostException { HostProvider hostProvider = getHostProvider((byte) 4); // 10.10.10.4:1238, 10.10.10.3:1237, 10.10.10.2:1236, 10.10.10.1:1235 - Collection newList = getServerAddresses((byte) 3); // 10.10.10.3:1237, 10.10.10.2:1236, 10.10.10.1:1235 + Collection newAddrList = getServerAddresses((byte) 3); // 10.10.10.3:1237, 10.10.10.2:1236, 10.10.10.1:1235 + Collection newList = inetToServerCfgList(newAddrList); InetSocketAddress myServer = new InetSocketAddress(InetAddress.getByAddress(new byte[]{10, 10, 10, 3}), 1237); @@ -133,7 +140,9 @@ public void testUpdateClientMigrateOrNot() throws UnknownHostException { // Number of machines became smaller, my server is not in the new // cluster - newList = getServerAddresses((byte) 2); // 10.10.10.2:1236, 10.10.10.1:1235 + newAddrList = getServerAddresses((byte) 2); // 10.10.10.2:1236, 10.10.10.1:1235 + newList = inetToServerCfgList(newAddrList); + disconnectRequired = hostProvider.updateServerList(newList, myServer); assertTrue(disconnectRequired); hostProvider.onConnected(); @@ -145,9 +154,11 @@ public void testUpdateClientMigrateOrNot() throws UnknownHostException { hostProvider.onConnected(); // Number of machines increased, my server is not in the new cluster - newList = new ArrayList(3); + newAddrList = new ArrayList(3); + newList = new ArrayList(); for (byte i = 4; i > 1; i--) { // 10.10.10.4:1238, 10.10.10.3:1237, 10.10.10.2:1236 - newList.add(new InetSocketAddress(InetAddress.getByAddress(new byte[]{10, 10, 10, i}), 1234 + i)); + final InetSocketAddress addr = new InetSocketAddress(InetAddress.getByAddress(new byte[]{10, 10, 10, i}), 1234 + i); + newList.add(new ServerCfg(addr.getHostString(), addr)); } myServer = new InetSocketAddress(InetAddress.getByAddress(new byte[]{10, 10, 10, 1}), 1235); disconnectRequired = hostProvider.updateServerList(newList, myServer); @@ -160,7 +171,9 @@ public void testUpdateClientMigrateOrNot() throws UnknownHostException { // With probability 1 - |old|/|new} the client disconnects // In the test below 1-9/10 = 1/10 chance of disconnecting HostProvider[] hostProviderArray = new HostProvider[numClients]; - newList = getServerAddresses((byte) 10); + newAddrList = getServerAddresses((byte) 10); + newList = inetToServerCfgList(newAddrList); + int numDisconnects = 0; for (int i = 0; i < numClients; i++) { hostProviderArray[i] = getHostProvider((byte) 9); @@ -201,8 +214,8 @@ public void testUpdateMigrationGoesRound() throws UnknownHostException { // load on old servers must be decreased, so must connect to one of the // new servers // i.e., pNew = 1. - - boolean disconnectRequired = hostProvider.updateServerList(newList, new InetSocketAddress(InetAddress.getByAddress(new byte[]{10, 10, 10, 1}), 1235)); + Collection serverCfgList = inetToServerCfgList(newList); + boolean disconnectRequired = hostProvider.updateServerList(serverCfgList, new InetSocketAddress(InetAddress.getByAddress(new byte[]{10, 10, 10, 1}), 1235)); assertTrue(disconnectRequired); // This means reconfigMode = true, and nextHostInReconfigMode will be @@ -210,30 +223,30 @@ public void testUpdateMigrationGoesRound() throws UnknownHostException { // Since pNew = 1 we should first try the new servers ArrayList seen = new ArrayList(); for (int i = 0; i < newComing.size(); i++) { - InetSocketAddress addr = hostProvider.next(0); - assertTrue(newComing.contains(addr)); - assertTrue(!seen.contains(addr)); - seen.add(addr); + ServerCfg serverCfg = hostProvider.next(0); + assertTrue(newComing.contains(serverCfg.getInetAddress())); + assertTrue(!seen.contains(serverCfg.getInetAddress())); + seen.add(serverCfg.getInetAddress()); } // Next the old servers seen.clear(); for (int i = 0; i < oldStaying.size(); i++) { - InetSocketAddress addr = hostProvider.next(0); - assertTrue(oldStaying.contains(addr)); - assertTrue(!seen.contains(addr)); - seen.add(addr); + ServerCfg serverCfg = hostProvider.next(0); + assertTrue(oldStaying.contains(serverCfg.getInetAddress())); + assertTrue(!seen.contains(serverCfg.getInetAddress())); + seen.add(serverCfg.getInetAddress()); } // And now it goes back to normal next() so it should be everything // together like in testNextGoesRound() - InetSocketAddress first = hostProvider.next(0); - assertTrue(first != null); + ServerCfg serverCfg = hostProvider.next(0); + assertTrue(serverCfg != null); for (int i = 0; i < newList.size() - 1; i++) { hostProvider.next(0); } - assertEquals(first, hostProvider.next(0)); + assertEquals(serverCfg, hostProvider.next(0)); hostProvider.onConnected(); } @@ -242,7 +255,7 @@ public void testUpdateLoadBalancing() throws UnknownHostException { // Start with 9 servers and 10000 clients boolean disconnectRequired; HostProvider[] hostProviderArray = new HostProvider[numClients]; - InetSocketAddress[] curHostForEachClient = new InetSocketAddress[numClients]; + ServerCfg[] curHostForEachClient = new ServerCfg[numClients]; int[] numClientsPerHost = new int[9]; // initialization @@ -261,9 +274,9 @@ public void testUpdateLoadBalancing() throws UnknownHostException { // remove host number 8 (the last one in a list of 9 hosts) Collection newList = getServerAddresses((byte) 8); - + Collection serverCfgList = inetToServerCfgList(newList); for (int i = 0; i < numClients; i++) { - disconnectRequired = hostProviderArray[i].updateServerList(newList, curHostForEachClient[i]); + disconnectRequired = hostProviderArray[i].updateServerList(serverCfgList, curHostForEachClient[i].getInetAddress()); if (disconnectRequired) curHostForEachClient[i] = hostProviderArray[i].next(0); numClientsPerHost[curHostForEachClient[i].getPort() - 1235]++; hostProviderArray[i].onConnected(); @@ -278,9 +291,9 @@ public void testUpdateLoadBalancing() throws UnknownHostException { // remove hosts number 6 and 7 (the currently last two in the list) newList = getServerAddresses((byte) 6); - + serverCfgList = inetToServerCfgList(newList); for (int i = 0; i < numClients; i++) { - disconnectRequired = hostProviderArray[i].updateServerList(newList, curHostForEachClient[i]); + disconnectRequired = hostProviderArray[i].updateServerList(serverCfgList, curHostForEachClient[i].getInetAddress()); if (disconnectRequired) curHostForEachClient[i] = hostProviderArray[i].next(0); numClientsPerHost[curHostForEachClient[i].getPort() - 1235]++; hostProviderArray[i].onConnected(); @@ -301,9 +314,9 @@ public void testUpdateLoadBalancing() throws UnknownHostException { for (byte i = 9; i > 1; i--) { newList.add(new InetSocketAddress(InetAddress.getByAddress(new byte[]{10, 10, 10, i}), 1234 + i)); } - + serverCfgList = inetToServerCfgList(newList); for (int i = 0; i < numClients; i++) { - disconnectRequired = hostProviderArray[i].updateServerList(newList, curHostForEachClient[i]); + disconnectRequired = hostProviderArray[i].updateServerList(serverCfgList, curHostForEachClient[i].getInetAddress()); if (disconnectRequired) curHostForEachClient[i] = hostProviderArray[i].next(0); numClientsPerHost[curHostForEachClient[i].getPort() - 1235]++; hostProviderArray[i].onConnected(); @@ -319,9 +332,9 @@ public void testUpdateLoadBalancing() throws UnknownHostException { // add back host number 0 newList = getServerAddresses((byte) 9); - + serverCfgList = inetToServerCfgList(newList); for (int i = 0; i < numClients; i++) { - disconnectRequired = hostProviderArray[i].updateServerList(newList, curHostForEachClient[i]); + disconnectRequired = hostProviderArray[i].updateServerList(serverCfgList, curHostForEachClient[i].getInetAddress()); if (disconnectRequired) curHostForEachClient[i] = hostProviderArray[i].next(0); numClientsPerHost[curHostForEachClient[i].getPort() - 1235]++; hostProviderArray[i].onConnected(); @@ -338,7 +351,7 @@ public void testNoCurrentHostDuringNormalMode() throws UnknownHostException { // Start with 9 servers and 10000 clients boolean disconnectRequired; StaticHostProvider[] hostProviderArray = new StaticHostProvider[numClients]; - InetSocketAddress[] curHostForEachClient = new InetSocketAddress[numClients]; + ServerCfg[] curHostForEachClient = new ServerCfg[numClients]; int[] numClientsPerHost = new int[9]; // initialization @@ -355,12 +368,12 @@ public void testNoCurrentHostDuringNormalMode() throws UnknownHostException { // remove hosts 7 and 8 (the last two in a list of 9 hosts) Collection newList = getServerAddresses((byte) 7); - + Collection serverCfgList = inetToServerCfgList(newList); for (int i = 0; i < numClients; i++) { // tests the case currentHost == null && lastIndex == -1 // calls next for clients with index < numClients/2 - disconnectRequired = hostProviderArray[i].updateServerList(newList, - curHostForEachClient[i]); + disconnectRequired = hostProviderArray[i].updateServerList(serverCfgList, + curHostForEachClient[i].getInetAddress()); if (disconnectRequired) curHostForEachClient[i] = hostProviderArray[i].next(0); else if (curHostForEachClient[i] == null) { @@ -383,12 +396,12 @@ else if (curHostForEachClient[i] == null) { // add back server 7 newList = getServerAddresses((byte) 8); - + serverCfgList = inetToServerCfgList(newList); for (int i = 0; i < numClients; i++) { InetSocketAddress myServer = (i < (numClients / 2)) ? null - : curHostForEachClient[i]; + : curHostForEachClient[i].getInetAddress(); // tests the case currentHost == null && lastIndex >= 0 - disconnectRequired = hostProviderArray[i].updateServerList(newList, + disconnectRequired = hostProviderArray[i].updateServerList(serverCfgList, myServer); if (disconnectRequired) curHostForEachClient[i] = hostProviderArray[i].next(0); @@ -407,7 +420,7 @@ public void testReconfigDuringReconfigMode() throws UnknownHostException { // Start with 9 servers and 10000 clients boolean disconnectRequired; StaticHostProvider[] hostProviderArray = new StaticHostProvider[numClients]; - InetSocketAddress[] curHostForEachClient = new InetSocketAddress[numClients]; + ServerCfg[] curHostForEachClient = new ServerCfg[numClients]; int[] numClientsPerHost = new int[9]; // initialization @@ -418,25 +431,25 @@ public void testReconfigDuringReconfigMode() throws UnknownHostException { // remove hosts 7 and 8 (the last two in a list of 9 hosts) Collection newList = getServerAddresses((byte) 7); - + Collection serverCfgList = inetToServerCfgList(newList); for (int i = 0; i < numClients; i++) { // sets reconfigMode - hostProviderArray[i].updateServerList(newList, - curHostForEachClient[i]); + hostProviderArray[i].updateServerList(serverCfgList, + curHostForEachClient[i].getInetAddress()); } // add back servers 7 and 8 while still in reconfigMode (we didn't call // next) newList = getServerAddresses((byte) 9); - + serverCfgList = inetToServerCfgList(newList); for (int i = 0; i < numClients; i++) { InetSocketAddress myServer = (i < (numClients / 2)) ? null - : curHostForEachClient[i]; + : curHostForEachClient[i].getInetAddress(); // for i < (numClients/2) this tests the case currentHost == null && // reconfigMode = true // for i >= (numClients/2) this tests the case currentHost!=null && // reconfigMode = true - disconnectRequired = hostProviderArray[i].updateServerList(newList, + disconnectRequired = hostProviderArray[i].updateServerList(serverCfgList, myServer); if (disconnectRequired) curHostForEachClient[i] = hostProviderArray[i].next(0); @@ -457,7 +470,7 @@ public void testReconfigDuringReconfigMode() throws UnknownHostException { } private StaticHostProvider getHostProvider(byte size) { - return new StaticHostProvider(getServerAddresses(size), r.nextLong()); + return new StaticHostProvider(inetToServerCfgList(getServerAddresses(size)), r.nextLong()); } private HashMap> precomputedLists = new @@ -492,20 +505,20 @@ public void testLiteralIPNoReverseNS() throws Exception { byte size = 30; HostProvider hostProvider = getHostProviderUnresolved(size); for (int i = 0; i < size; i++) { - InetSocketAddress next = hostProvider.next(0); - assertTrue(next instanceof InetSocketAddress); - assertTrue(!next.isUnresolved()); + ServerCfg next = hostProvider.next(0); + assertTrue(next instanceof ServerCfg); + assertTrue(!next.getInetAddress().isUnresolved()); assertTrue(!next.toString().startsWith("/")); // Do NOT trigger the reverse name service lookup. String hostname = next.getHostString(); // In this case, the hostname equals literal IP address. - hostname.equals(next.getAddress().getHostAddress()); + hostname.equals(next.getInetAddress().getAddress().getHostAddress()); } } private StaticHostProvider getHostProviderUnresolved(byte size) throws UnknownHostException { - return new StaticHostProvider(getUnresolvedServerAddresses(size), r.nextLong()); + return new StaticHostProvider(inetToServerCfgList(getUnresolvedServerAddresses(size)), r.nextLong()); } private Collection getUnresolvedServerAddresses(byte size) { @@ -516,4 +529,14 @@ private Collection getUnresolvedServerAddresses(byte size) { } return list; } + + private Collection inetToServerCfgList( + final Collection inetSocketAddresses) { + final Collection serverCfgList = new ArrayList(); + for (final InetSocketAddress addr : inetSocketAddresses) { + serverCfgList.add(new ServerCfg(addr.getHostString(), addr)); + } + + return serverCfgList; + } } From b249a1d79d7f46ddba3855922348cc00e2c18ad7 Mon Sep 17 00:00:00 2001 From: Powell Molleti Date: Wed, 17 Aug 2016 22:04:21 -0700 Subject: [PATCH 02/12] Pass ZKConfig() around to make things work. --- .../main/org/apache/zookeeper/SSLCertCfg.java | 4 +- .../main/org/apache/zookeeper/ZooKeeper.java | 30 +++++++-- .../org/apache/zookeeper/common/X509Util.java | 61 +++++++++++++------ .../org/apache/zookeeper/common/ZKConfig.java | 13 ++++ .../zookeeper/server/quorum/QuorumPeer.java | 4 +- .../quorum/util/ZKPeerX509TrustManager.java | 3 +- 6 files changed, 87 insertions(+), 28 deletions(-) diff --git a/src/java/main/org/apache/zookeeper/SSLCertCfg.java b/src/java/main/org/apache/zookeeper/SSLCertCfg.java index 7ac74a62f05..71db5757308 100644 --- a/src/java/main/org/apache/zookeeper/SSLCertCfg.java +++ b/src/java/main/org/apache/zookeeper/SSLCertCfg.java @@ -23,6 +23,7 @@ import java.util.Map; import org.apache.zookeeper.common.X509Util; +import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -117,7 +118,8 @@ private static MessageDigest getMessageDigest(final String fp) throws QuorumPeerConfig.ConfigException { // Check for supported algos for the given fingerprint if cannot // validate throw exception. - MessageDigest md = X509Util.getSupportedMessageDigestForFpStr(fp); + MessageDigest md = X509Util.getSupportedMessageDigestForFpStr( + new ZKConfig(), fp); if (md == null) { final String errStr = "Algo in fingerprint: " + fp + " not supported, bailing out"; diff --git a/src/java/main/org/apache/zookeeper/ZooKeeper.java b/src/java/main/org/apache/zookeeper/ZooKeeper.java index e5d0fcd71f2..95f342b75e8 100644 --- a/src/java/main/org/apache/zookeeper/ZooKeeper.java +++ b/src/java/main/org/apache/zookeeper/ZooKeeper.java @@ -154,10 +154,6 @@ public class ZooKeeper implements AutoCloseable { @Deprecated public static final String SECURE_CLIENT = "zookeeper.client.secure"; - // TODO: XXX: Move to ZKConfig - public static final String PEER_HOST_CERT_FINGERPRINT - = "zookeeper.client.peer.cert.fingerprint"; - protected final ClientCnxn cnxn; private static final Logger LOG; static { @@ -670,6 +666,9 @@ public boolean isConnected() { * would be relative to this root - ie getting/setting/etc... * "/foo/bar" would result in operations being run on * "/app/a/foo/bar" (from the server perspective). + * With SSL support the string might look like this: + * "127.0.0.1:3000:SHA-256-XXXXX,127.0.0.1:3001:SHA-256-XXXXX, + * 127.0.0.1:3002:SHA-256-XXXXX". * @param sessionTimeout * session timeout in milliseconds * @param watcher @@ -719,6 +718,9 @@ public ZooKeeper(String connectString, int sessionTimeout, Watcher watcher) * would be relative to this root - ie getting/setting/etc... * "/foo/bar" would result in operations being run on * "/app/a/foo/bar" (from the server perspective). + * With SSL support the string might look like this: + * "127.0.0.1:3000:SHA-256-XXXXX,127.0.0.1:3001:SHA-256-XXXXX, + * 127.0.0.1:3002:SHA-256-XXXXX". * @param sessionTimeout * session timeout in milliseconds * @param watcher @@ -773,6 +775,9 @@ public ZooKeeper(String connectString, int sessionTimeout, Watcher watcher, * would be relative to this root - ie getting/setting/etc... * "/foo/bar" would result in operations being run on * "/app/a/foo/bar" (from the server perspective). + * With SSL support the string might look like this: + * "127.0.0.1:3000:SHA-256-XXXXX,127.0.0.1:3001:SHA-256-XXXXX, + * 127.0.0.1:3002:SHA-256-XXXXX". * @param sessionTimeout * session timeout in milliseconds * @param watcher @@ -838,6 +843,9 @@ public ZooKeeper(String connectString, int sessionTimeout, Watcher watcher, * would be relative to this root - ie getting/setting/etc... * "/foo/bar" would result in operations being run on * "/app/a/foo/bar" (from the server perspective). + * With SSL support the string might look like this: + * "127.0.0.1:3000:SHA-256-XXXXX,127.0.0.1:3001:SHA-256-XXXXX, + * 127.0.0.1:3002:SHA-256-XXXXX". * @param sessionTimeout * session timeout in milliseconds * @param watcher @@ -916,6 +924,9 @@ public ZooKeeper(String connectString, int sessionTimeout, Watcher watcher, * would be relative to this root - ie getting/setting/etc... * "/foo/bar" would result in operations being run on * "/app/a/foo/bar" (from the server perspective). + * With SSL support the string might look like this: + * "127.0.0.1:3000:SHA-256-XXXXX,127.0.0.1:3001:SHA-256-XXXXX, + * 127.0.0.1:3002:SHA-256-XXXXX". * @param sessionTimeout * session timeout in milliseconds * @param watcher @@ -974,6 +985,9 @@ public ZooKeeper(String connectString, int sessionTimeout, Watcher watcher, * would be relative to this root - ie getting/setting/etc... * "/foo/bar" would result in operations being run on * "/app/a/foo/bar" (from the server perspective). + * With SSL support the string might look like this: + * "127.0.0.1:3000:SHA-256-XXXXX,127.0.0.1:3001:SHA-256-XXXXX, + * 127.0.0.1:3002:SHA-256-XXXXX". * @param sessionTimeout * session timeout in milliseconds * @param watcher @@ -1039,6 +1053,9 @@ public ZooKeeper(String connectString, int sessionTimeout, Watcher watcher, * would be relative to this root - ie getting/setting/etc... * "/foo/bar" would result in operations being run on * "/app/a/foo/bar" (from the server perspective). + * With SSL support the string might look like this: + * "127.0.0.1:3000:SHA-256-XXXXX,127.0.0.1:3001:SHA-256-XXXXX, + * 127.0.0.1:3002:SHA-256-XXXXX". * @param sessionTimeout * session timeout in milliseconds * @param watcher @@ -1102,6 +1119,9 @@ public ZooKeeper(String connectString, int sessionTimeout, Watcher watcher, * would be relative to this root - ie getting/setting/etc... * "/foo/bar" would result in operations being run on * "/app/a/foo/bar" (from the server perspective). + * With SSL support the string might look like this: + * "127.0.0.1:3000:SHA-256-XXXXX,127.0.0.1:3001:SHA-256-XXXXX, + * 127.0.0.1:3002:SHA-256-XXXXX". * @param sessionTimeout * session timeout in milliseconds * @param watcher @@ -1192,7 +1212,7 @@ public ZooKeeper(String connectString, int sessionTimeout, Watcher watcher, * "/app/a/foo/bar" (from the server perspective). * With SSL support the string might look like this: * "127.0.0.1:3000:SHA-256-XXXXX,127.0.0.1:3001:SHA-256-XXXXX, - * 127.0.0.1:3002:SHA-256-XXXXX" + * 127.0.0.1:3002:SHA-256-XXXXX". * @param sessionTimeout * session timeout in milliseconds * @param watcher diff --git a/src/java/main/org/apache/zookeeper/common/X509Util.java b/src/java/main/org/apache/zookeeper/common/X509Util.java index c34af092ddb..56c3641472c 100644 --- a/src/java/main/org/apache/zookeeper/common/X509Util.java +++ b/src/java/main/org/apache/zookeeper/common/X509Util.java @@ -63,9 +63,16 @@ public class X509Util { private static final Logger LOG = LoggerFactory.getLogger(X509Util.class); /** - * TODO: XXX: Move to ZKClient + * @deprecated Use {@link ZKConfig#SSL_VERSION_DEFAULT} + * instead. */ + @Deprecated public static final String SSL_VERSION_DEFAULT = "TLSv1"; + /** + * @deprecated Use {@link ZKConfig#SSL_VERSION} + * instead. + */ + @Deprecated public static final String SSL_VERSION = "zookeeper.ssl.version"; /** * @deprecated Use {@link ZKConfig#SSL_KEYSTORE_LOCATION} @@ -99,11 +106,23 @@ public class X509Util { public static final String SSL_AUTHPROVIDER = "zookeeper.ssl.authProvider"; /** - * TODO: XXX: Move to ZKClient + * @deprecated Use {@link ZKConfig#SSL_TRUSTSTORE_CA_ALIAS} + * instead. */ + @Deprecated public static final String SSL_TRUSTSTORE_CA_ALIAS = "zookeeper.ssl.trustStore.rootCA.alias"; + /** + * @deprecated Use {@link ZKConfig#SSL_DIGEST_DEFAULT_ALGO} + * instead. + */ + @Deprecated public static final String SSL_DIGEST_DEFAULT_ALGO ="SHA-256"; + /** + * @deprecated Use {@link ZKConfig#SSL_DIGEST_ALGOS} + * instead. + */ + @Deprecated public static final String SSL_DIGEST_ALGOS = "quorum.ssl.digest.algos"; /** @@ -157,9 +176,9 @@ public static SSLContext createSSLContext(final ZKConfig config, private static KeyManager[] createKeyManagers(final ZKConfig config) throws SSLContextException { final String keyStoreLocationProp = - config.getProperty(SSL_KEYSTORE_LOCATION); + config.getProperty(ZKConfig.SSL_KEYSTORE_LOCATION); final String keyStorePasswordProp = - config.getProperty(SSL_KEYSTORE_PASSWD); + config.getProperty(ZKConfig.SSL_KEYSTORE_PASSWD); if (keyStoreLocationProp == null && keyStorePasswordProp == null) { LOG.warn("keystore not specified for client connection"); @@ -194,9 +213,9 @@ private static TrustManager[] createTrustManagers( final ZKConfig config, final QuorumPeer quorumPeer) throws SSLContextException { String trustStoreLocationProp = - config.getProperty(SSL_TRUSTSTORE_LOCATION); + config.getProperty(ZKConfig.SSL_TRUSTSTORE_LOCATION); String trustStorePasswordProp = - config.getProperty(SSL_TRUSTSTORE_PASSWD); + config.getProperty(ZKConfig.SSL_TRUSTSTORE_PASSWD); if (trustStoreLocationProp == null && trustStorePasswordProp == null) { if (quorumPeer == null) { @@ -241,9 +260,9 @@ private static SSLContext createSSLContext( final KeyManager[] keyManagers, final TrustManager[] trustManagers) throws SSLContextException { - String sslVersion = config.getProperty(SSL_VERSION); + String sslVersion = config.getProperty(ZKConfig.SSL_VERSION); if (sslVersion == null) { - sslVersion = SSL_VERSION_DEFAULT; + sslVersion = ZKConfig.SSL_VERSION_DEFAULT; } SSLContext sslContext; try { @@ -304,7 +323,7 @@ public static X509TrustManager createTrustManager( final String trustStoreLocation, final String trustStorePassword) throws TrustManagerException, SSLContextException { String trustStoreCAAlias = - config.getProperty(SSL_TRUSTSTORE_CA_ALIAS); + config.getProperty(ZKConfig.SSL_TRUSTSTORE_CA_ALIAS); if (trustStoreCAAlias == null) { final String errStr = "No CA Alias provided, need one to work in " + "CA mode."; @@ -366,12 +385,13 @@ private static X509Certificate getCertWithAlias( /** * Parse parsed system property and find a valid algo that matches * the finger print passed. Will return null if it couldn't - * @param fingerPrint + * @param config ZKConfig + * @param fingerPrint Digest of cert * @return MessageDigest object, null on error. */ public static MessageDigest getSupportedMessageDigestForFpStr( - final String fingerPrint) { - final String[] algos = getConfigureDigestAlgos(); + final ZKConfig config, final String fingerPrint) { + final String[] algos = getConfigureDigestAlgos(config); String validAlgo = null; for (int i = 0; i < algos.length; i++) { @@ -389,7 +409,7 @@ public static MessageDigest getSupportedMessageDigestForFpStr( return null; } - MessageDigest md = getMessageDigestByAlgo(validAlgo); + MessageDigest md = getMessageDigestByAlgo(config, validAlgo); if (md == null) { return null; } @@ -413,24 +433,24 @@ public static MessageDigest getSupportedMessageDigestForFpStr( return md; } - private static String[] getConfigureDigestAlgos() { - String digest_algos = System.getProperty(SSL_DIGEST_ALGOS); + private static String[] getConfigureDigestAlgos(final ZKConfig config) { + String digest_algos = config.getProperty(ZKConfig.SSL_DIGEST_ALGOS); if (digest_algos == null) { - digest_algos = SSL_DIGEST_DEFAULT_ALGO; + digest_algos = ZKConfig.SSL_DIGEST_DEFAULT_ALGO; } return digest_algos.trim().toLowerCase().split(","); } private static MessageDigest getMessageDigestByAlgo( - final String validAlgo) { + final ZKConfig config, final String validAlgo) { MessageDigest md = null; try { LOG.info("Valid algo: " + validAlgo); md = MessageDigest.getInstance(validAlgo.toUpperCase()); } catch (NoSuchAlgorithmException e) { LOG.error("Invalid algo: " + validAlgo + " support algos: " + - getConfigureDigestAlgos()); + String.join(",", getConfigureDigestAlgos(config))); } return md; @@ -444,11 +464,12 @@ private static MessageDigest getMessageDigestByAlgo( * @return True on success * @throws CertificateEncodingException */ - public static boolean validateCert(final String fingerPrint, + public static boolean validateCert(final ZKConfig config, + final String fingerPrint, final X509Certificate cert) throws CertificateEncodingException, NoSuchAlgorithmException { final MessageDigest fpMsgDigest = - getSupportedMessageDigestForFpStr(fingerPrint); + getSupportedMessageDigestForFpStr(config, fingerPrint); if (fpMsgDigest == null) { return false; } diff --git a/src/java/main/org/apache/zookeeper/common/ZKConfig.java b/src/java/main/org/apache/zookeeper/common/ZKConfig.java index 8d9c001328d..c3ff34fd87b 100644 --- a/src/java/main/org/apache/zookeeper/common/ZKConfig.java +++ b/src/java/main/org/apache/zookeeper/common/ZKConfig.java @@ -43,6 +43,10 @@ public class ZKConfig { private static final Logger LOG = LoggerFactory.getLogger(ZKConfig.class); @SuppressWarnings("deprecation") + public static final String SSL_VERSION_DEFAULT = X509Util.SSL_VERSION_DEFAULT; + @SuppressWarnings("deprecation") + public static final String SSL_VERSION = X509Util.SSL_VERSION; + @SuppressWarnings("deprecation") public static final String SSL_KEYSTORE_LOCATION = X509Util.SSL_KEYSTORE_LOCATION; @SuppressWarnings("deprecation") public static final String SSL_KEYSTORE_PASSWD = X509Util.SSL_KEYSTORE_PASSWD; @@ -52,6 +56,12 @@ public class ZKConfig { public static final String SSL_TRUSTSTORE_PASSWD = X509Util.SSL_TRUSTSTORE_PASSWD; @SuppressWarnings("deprecation") public static final String SSL_AUTHPROVIDER = X509Util.SSL_AUTHPROVIDER; + @SuppressWarnings("deprecation") + public static final String SSL_TRUSTSTORE_CA_ALIAS = X509Util.SSL_TRUSTSTORE_CA_ALIAS; + @SuppressWarnings("deprecation") + public static final String SSL_DIGEST_DEFAULT_ALGO = X509Util.SSL_DIGEST_DEFAULT_ALGO; + @SuppressWarnings("deprecation") + public static final String SSL_DIGEST_ALGOS = X509Util.SSL_DIGEST_ALGOS; public static final String JUTE_MAXBUFFER = "jute.maxbuffer"; /** * Path to a kinit binary: {@value}. Defaults to @@ -107,11 +117,14 @@ private void init() { * this configuration. */ protected void handleBackwardCompatibility() { + properties.put(SSL_VERSION, System.getProperty(SSL_VERSION)); properties.put(SSL_KEYSTORE_LOCATION, System.getProperty(SSL_KEYSTORE_LOCATION)); properties.put(SSL_KEYSTORE_PASSWD, System.getProperty(SSL_KEYSTORE_PASSWD)); properties.put(SSL_TRUSTSTORE_LOCATION, System.getProperty(SSL_TRUSTSTORE_LOCATION)); properties.put(SSL_TRUSTSTORE_PASSWD, System.getProperty(SSL_TRUSTSTORE_PASSWD)); properties.put(SSL_AUTHPROVIDER, System.getProperty(SSL_AUTHPROVIDER)); + properties.put(SSL_TRUSTSTORE_CA_ALIAS, System.getProperty(SSL_TRUSTSTORE_CA_ALIAS)); + properties.put(SSL_DIGEST_ALGOS, System.getProperty(SSL_DIGEST_ALGOS)); properties.put(JUTE_MAXBUFFER, System.getProperty(JUTE_MAXBUFFER)); properties.put(KINIT_COMMAND, System.getProperty(KINIT_COMMAND)); properties.put(JGSS_NATIVE, System.getProperty(JGSS_NATIVE)); diff --git a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java index 2a239a8787f..c96e7120880 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java @@ -53,6 +53,7 @@ import org.apache.zookeeper.common.AtomicFileWritingIdiom.WriterStatement; import org.apache.zookeeper.common.Time; import org.apache.zookeeper.common.X509Util; +import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.jmx.MBeanRegistry; import org.apache.zookeeper.jmx.ZKMBeanInfo; import org.apache.zookeeper.server.ServerCnxnFactory; @@ -1653,7 +1654,8 @@ private String getQuorumServerFingerPrintByCert( final String peerFp = quorumServer.getSslCertCfg().getCertFingerPrintStr(); final MessageDigest peerFpSupportedMd = - X509Util.getSupportedMessageDigestForFpStr(peerFp); + X509Util.getSupportedMessageDigestForFpStr( + new ZKConfig(), peerFp); if (peerFpSupportedMd == null) { final String errStr = "QuorumServer: " + quorumServer + diff --git a/src/java/main/org/apache/zookeeper/server/quorum/util/ZKPeerX509TrustManager.java b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKPeerX509TrustManager.java index d08da78c029..a9c97f07bf2 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/util/ZKPeerX509TrustManager.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKPeerX509TrustManager.java @@ -25,6 +25,7 @@ import javax.net.ssl.X509TrustManager; import org.apache.zookeeper.common.X509Util; +import org.apache.zookeeper.common.ZKConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,7 +72,7 @@ private void validatePeerCert(final X509Certificate cert) X509Util.verifySelfSigned(cert); try { - X509Util.validateCert(peerCertFingerPrintStr, cert); + X509Util.validateCert(new ZKConfig(), peerCertFingerPrintStr, cert); } catch (NoSuchAlgorithmException exp) { throw new CertificateException(exp); } From 2f2c8cbaadf8db9c7aa4b51588d5c35547163990 Mon Sep 17 00:00:00 2001 From: Powell Molleti Date: Sun, 21 Aug 2016 13:00:45 -0700 Subject: [PATCH 03/12] Fix QuorumPeer constructor call. --- .../apache/zookeeper/server/quorum/RaceConditionTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/java/test/org/apache/zookeeper/server/quorum/RaceConditionTest.java b/src/java/test/org/apache/zookeeper/server/quorum/RaceConditionTest.java index c65a7944d49..d683759744d 100644 --- a/src/java/test/org/apache/zookeeper/server/quorum/RaceConditionTest.java +++ b/src/java/test/org/apache/zookeeper/server/quorum/RaceConditionTest.java @@ -35,6 +35,8 @@ import org.apache.zookeeper.server.ZooKeeperServer; import org.apache.zookeeper.server.persistence.FileTxnSnapLog; import org.apache.zookeeper.server.quorum.QuorumPeer.ServerState; +import org.apache.zookeeper.server.quorum.flexible.QuorumMaj; +import org.apache.zookeeper.server.quorum.util.QuorumSocketFactory; import org.apache.zookeeper.test.ClientBase; import org.junit.After; import org.junit.Assert; @@ -42,6 +44,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + /** * This test class contains test cases related to race condition in complete * ZooKeeper From 98cc6c48e0ead57be221c0bb3729f9f1bde37382 Mon Sep 17 00:00:00 2001 From: Powell Molleti Date: Sun, 21 Aug 2016 13:43:32 -0700 Subject: [PATCH 04/12] Fix a typo, pass the host ip part of the string. --- .../zookeeper/client/ConnectStringParser.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/java/main/org/apache/zookeeper/client/ConnectStringParser.java b/src/java/main/org/apache/zookeeper/client/ConnectStringParser.java index 1a23c60f1fe..47c7d1befae 100644 --- a/src/java/main/org/apache/zookeeper/client/ConnectStringParser.java +++ b/src/java/main/org/apache/zookeeper/client/ConnectStringParser.java @@ -28,6 +28,8 @@ import org.apache.zookeeper.ServerCfg; import org.apache.zookeeper.common.PathUtils; import org.apache.zookeeper.server.quorum.QuorumPeerConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static org.apache.zookeeper.common.StringUtils.split; @@ -42,6 +44,8 @@ * @see org.apache.zookeeper.ZooKeeper */ public final class ConnectStringParser { + private static final Logger LOG = LoggerFactory + .getLogger(ConnectStringParser.class); private static final int DEFAULT_PORT = 2181; private final String chrootPath; @@ -87,16 +91,17 @@ public ConnectStringParser(String connectString) { try { if (hostStrParts.length > 2 || noPort) { + serverCfgList.add( new ServerCfg(hostStrParts[0], - InetSocketAddress.createUnresolved(host, - port), + InetSocketAddress.createUnresolved( + hostStrParts[0], port), SSLCertCfg.parseCertCfgStr(host))); } else { serverCfgList.add( new ServerCfg(hostStrParts[0], - InetSocketAddress.createUnresolved(host, - port))); + InetSocketAddress.createUnresolved( + hostStrParts[0], port))); } } catch (QuorumPeerConfig.ConfigException exp) { throw new IllegalArgumentException(exp); From 8e35e86bd0de86b4378bcb8a7e0cb920f263b9a5 Mon Sep 17 00:00:00 2001 From: Powell Molleti Date: Sun, 21 Aug 2016 16:50:04 -0700 Subject: [PATCH 05/12] Make ZookeeperServer SSL auth to force client to be one of the ZK nodes. For now this is ok but this is not useful for everyone. TODO: Create a different trust manager system property for north-south SSL perhaps along with QuorumPeer dynamic verification. --- .../org/apache/zookeeper/common/X509Util.java | 2 +- .../server/NettyServerCnxnFactory.java | 23 ++++++++----------- .../auth/X509AuthenticationProvider.java | 11 ++++++--- .../util/ZKDynamicX509TrustManager.java | 8 +++---- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/java/main/org/apache/zookeeper/common/X509Util.java b/src/java/main/org/apache/zookeeper/common/X509Util.java index 56c3641472c..637a05e6afd 100644 --- a/src/java/main/org/apache/zookeeper/common/X509Util.java +++ b/src/java/main/org/apache/zookeeper/common/X509Util.java @@ -364,7 +364,7 @@ private static X509TrustManager createTrustManager( return new ZKX509TrustManager(rootCA); } - private static X509TrustManager createTrustManager( + public static X509TrustManager createTrustManager( final QuorumPeer quorumPeer) { return new ZKDynamicX509TrustManager(quorumPeer); } diff --git a/src/java/main/org/apache/zookeeper/server/NettyServerCnxnFactory.java b/src/java/main/org/apache/zookeeper/server/NettyServerCnxnFactory.java index 0b7aaed19dc..bec7c26bae5 100644 --- a/src/java/main/org/apache/zookeeper/server/NettyServerCnxnFactory.java +++ b/src/java/main/org/apache/zookeeper/server/NettyServerCnxnFactory.java @@ -38,7 +38,6 @@ import org.apache.zookeeper.common.X509Exception; import org.apache.zookeeper.common.X509Util; import org.apache.zookeeper.common.ZKConfig; -import org.apache.zookeeper.server.auth.ProviderRegistry; import org.apache.zookeeper.server.auth.X509AuthenticationProvider; import org.jboss.netty.bootstrap.ServerBootstrap; import org.jboss.netty.buffer.ChannelBuffer; @@ -106,8 +105,14 @@ public void channelConnected(ChannelHandlerContext ctx, zkServer, NettyServerCnxnFactory.this); ctx.setAttachment(cnxn); - allChannels.add(ctx.getChannel()); - addCnxn(cnxn); + if (secure) { + SslHandler sslHandler = ctx.getPipeline().get(SslHandler.class); + ChannelFuture handshakeFuture = sslHandler.handshake(); + handshakeFuture.addListener(new CertificateVerifier(sslHandler, cnxn)); + } else { + allChannels.add(ctx.getChannel()); + addCnxn(cnxn); + } } @Override @@ -282,18 +287,8 @@ public void operationComplete(ChannelFuture future) SSLSession session = eng.getSession(); cnxn.setClientCertificateChain(session.getPeerCertificates()); - String authProviderProp - = System.getProperty(ZKConfig.SSL_AUTHPROVIDER, "x509"); - X509AuthenticationProvider authProvider = - (X509AuthenticationProvider) - ProviderRegistry.getProvider(authProviderProp); - - if (authProvider == null) { - LOG.error("Auth provider not found: {}", authProviderProp); - cnxn.close(); - return; - } + new X509AuthenticationProvider(quorumPeer); if (KeeperException.Code.OK != authProvider.handleAuthentication(cnxn, null)) { diff --git a/src/java/main/org/apache/zookeeper/server/auth/X509AuthenticationProvider.java b/src/java/main/org/apache/zookeeper/server/auth/X509AuthenticationProvider.java index f6a1174d8c6..300fe7ee51f 100644 --- a/src/java/main/org/apache/zookeeper/server/auth/X509AuthenticationProvider.java +++ b/src/java/main/org/apache/zookeeper/server/auth/X509AuthenticationProvider.java @@ -33,6 +33,7 @@ import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.data.Id; import org.apache.zookeeper.server.ServerCnxn; +import org.apache.zookeeper.server.quorum.QuorumPeer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,7 +66,7 @@ public class X509AuthenticationProvider implements AuthenticationProvider { *
zookeeper.ssl.keyStore.password *
zookeeper.ssl.trustStore.password */ - public X509AuthenticationProvider() { + public X509AuthenticationProvider(final QuorumPeer quorumPeer) { String keyStoreLocationProp = System.getProperty( ZKConfig.SSL_KEYSTORE_LOCATION); String keyStorePasswordProp = System.getProperty( @@ -86,8 +87,12 @@ public X509AuthenticationProvider() { ZKConfig.SSL_TRUSTSTORE_PASSWD); try { - tm = X509Util.createTrustManager(new ZKConfig(), - trustStoreLocationProp, trustStorePasswordProp); + if (quorumPeer != null) { + tm = X509Util.createTrustManager(quorumPeer); + } else { + tm = X509Util.createTrustManager(new ZKConfig(), + trustStoreLocationProp, trustStorePasswordProp); + } } catch (TrustManagerException | X509Exception.SSLContextException e) { LOG.error("Failed to create trust manager", e); throw new IllegalAccessError("Failed to create trust manager"); diff --git a/src/java/main/org/apache/zookeeper/server/quorum/util/ZKDynamicX509TrustManager.java b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKDynamicX509TrustManager.java index f05c226320d..5839457e63d 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/util/ZKDynamicX509TrustManager.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKDynamicX509TrustManager.java @@ -47,13 +47,13 @@ public ZKDynamicX509TrustManager(final QuorumPeer quorumPeer) { @Override public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { - + validateSingleAndSelfSigned(x509Certificates); } @Override public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { - + validateSingleAndSelfSigned(x509Certificates); } public X509Certificate[] getAcceptedIssuers() { @@ -78,14 +78,14 @@ public void checkServerTrusted( public void checkClientTrusted(X509Certificate[] x509Certificates, String s, SSLEngine sslEngine) throws CertificateException { - + validateSingleAndSelfSigned(x509Certificates); } @Override public void checkServerTrusted(X509Certificate[] x509Certificates, String s, SSLEngine sslEngine) throws CertificateException { - + validateSingleAndSelfSigned(x509Certificates); } private void validateSingleAndSelfSigned(final X509Certificate[] certs) From a040e08cb23ec16456277a6cc3e12566e15b369f Mon Sep 17 00:00:00 2001 From: Powell Molleti Date: Mon, 5 Sep 2016 15:03:41 -0700 Subject: [PATCH 06/12] Seperate SSL config for client and quorum Quorum servers will have different properties for SSL config, example: -Dquorum.ssl.enabled=true -Dquorum.ssl.keyStore.location=/root/zookeeper/ssl/testKeyStore.jks -Dquorum.ssl.keyStore.password=testpass -Dquorum.ssl.trustStore.location=/root/zookeeper/ssl/testTrustStore.jks -Dquorum.ssl.trustStore.password=testpass Enable each zookeeper node will be able to also allowed to be authenticated as a client using dynamic reconfig. Basic algorithm for various SSL connections. Client: 1. Use the given truststore if available 2. Use the given server's digest Server: 1. Use the given truststore if available 2. Use dynamic config to check against each server's digest Quorum: 1. Use the given truststore if available 2. Use dyanmic config to check against each server's digest --- .../apache/zookeeper/ClientCnxnSocket.java | 9 +- .../zookeeper/ClientCnxnSocketNetty.java | 37 +-- src/java/main/org/apache/zookeeper/Login.java | 19 +- .../main/org/apache/zookeeper/SSLCertCfg.java | 3 +- .../zookeeper/client/ClientX509Util.java | 57 +++++ .../zookeeper/client/FourLetterWordMain.java | 6 +- .../zookeeper/client/ZKClientConfig.java | 4 +- .../zookeeper/client/ZooKeeperSaslClient.java | 3 +- .../apache/zookeeper/common/SslConfig.java | 63 +++++ .../common/X509ChainedTrustManager.java | 218 ++++++++++++++++ .../org/apache/zookeeper/common/X509Util.java | 239 ++---------------- .../org/apache/zookeeper/common/ZKConfig.java | 80 +++--- .../server/NettyServerCnxnFactory.java | 21 +- .../zookeeper/server/ServerCnxnFactory.java | 2 +- .../server/ZookeeperServerConfig.java | 69 +++++ .../server/ZookeeperServerSslConfig.java | 36 +++ .../auth/X509AuthenticationProvider.java | 41 +-- .../zookeeper/server/quorum/QuorumPeer.java | 2 +- .../server/quorum/QuorumPeerConfig.java | 41 ++- .../server/quorum/QuorumSslConfig.java | 30 +++ .../quorum/util/QuorumSocketFactory.java | 19 +- .../server/quorum/util/QuorumX509Util.java | 68 +++++ .../quorum/util/ZKPeerX509TrustManager.java | 73 +++++- .../quorum/util/ZKX509TrustManager.java | 149 +++++++++-- .../zookeeper/server/util/ServerX509Util.java | 64 +++++ 25 files changed, 950 insertions(+), 403 deletions(-) create mode 100644 src/java/main/org/apache/zookeeper/client/ClientX509Util.java create mode 100644 src/java/main/org/apache/zookeeper/common/SslConfig.java create mode 100644 src/java/main/org/apache/zookeeper/common/X509ChainedTrustManager.java create mode 100644 src/java/main/org/apache/zookeeper/server/ZookeeperServerConfig.java create mode 100644 src/java/main/org/apache/zookeeper/server/ZookeeperServerSslConfig.java create mode 100644 src/java/main/org/apache/zookeeper/server/quorum/QuorumSslConfig.java create mode 100644 src/java/main/org/apache/zookeeper/server/quorum/util/QuorumX509Util.java create mode 100644 src/java/main/org/apache/zookeeper/server/util/ServerX509Util.java diff --git a/src/java/main/org/apache/zookeeper/ClientCnxnSocket.java b/src/java/main/org/apache/zookeeper/ClientCnxnSocket.java index b78b0cded8e..e1ca40262b6 100644 --- a/src/java/main/org/apache/zookeeper/ClientCnxnSocket.java +++ b/src/java/main/org/apache/zookeeper/ClientCnxnSocket.java @@ -29,7 +29,6 @@ import org.apache.zookeeper.ClientCnxn.Packet; import org.apache.zookeeper.client.ZKClientConfig; import org.apache.zookeeper.common.Time; -import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.proto.ConnectResponse; import org.apache.zookeeper.server.ByteBufferInputStream; import org.slf4j.Logger; @@ -230,15 +229,15 @@ abstract void doTransport(int waitTimeOut, List pendingQueue, protected void initProperties() throws IOException { try { - packetLen = clientConfig.getInt(ZKConfig.JUTE_MAXBUFFER, + packetLen = clientConfig.getInt(ZKClientConfig.JUTE_MAXBUFFER, ZKClientConfig.CLIENT_MAX_PACKET_LENGTH_DEFAULT); - LOG.info("{} value is {} Bytes", ZKConfig.JUTE_MAXBUFFER, + LOG.info("{} value is {} Bytes", ZKClientConfig.JUTE_MAXBUFFER, packetLen); } catch (NumberFormatException e) { String msg = MessageFormat.format( "Configured value {0} for property {1} can not be parsed to int", - clientConfig.getProperty(ZKConfig.JUTE_MAXBUFFER), - ZKConfig.JUTE_MAXBUFFER); + clientConfig.getProperty(ZKClientConfig.JUTE_MAXBUFFER), + ZKClientConfig.JUTE_MAXBUFFER); LOG.error(msg); throw new IOException(msg); } diff --git a/src/java/main/org/apache/zookeeper/ClientCnxnSocketNetty.java b/src/java/main/org/apache/zookeeper/ClientCnxnSocketNetty.java index 4a274adef5d..8576f3ebd42 100644 --- a/src/java/main/org/apache/zookeeper/ClientCnxnSocketNetty.java +++ b/src/java/main/org/apache/zookeeper/ClientCnxnSocketNetty.java @@ -18,10 +18,25 @@ package org.apache.zookeeper; +import java.io.IOException; +import java.net.SocketAddress; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; + import org.apache.zookeeper.ClientCnxn.EndOfStreamException; import org.apache.zookeeper.ClientCnxn.Packet; +import org.apache.zookeeper.client.ClientX509Util; import org.apache.zookeeper.client.ZKClientConfig; -import org.apache.zookeeper.common.X509Util; import org.jboss.netty.bootstrap.ClientBootstrap; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBuffers; @@ -42,21 +57,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLEngine; - -import java.io.IOException; -import java.net.SocketAddress; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; -import java.util.concurrent.Semaphore; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - import static org.apache.zookeeper.common.X509Exception.SSLContextException; /** @@ -366,8 +366,9 @@ public ChannelPipeline getPipeline() throws Exception { // Basically we only need to create it once. private synchronized void initSSL(ChannelPipeline pipeline) throws SSLContextException { if (sslContext == null || sslEngine == null) { - sslContext = X509Util.createSSLContext(clientConfig, - serverCfg.getInetAddress(), serverCfg.getSslCertCfg()); + sslContext = ClientX509Util.createSSLContext(clientConfig, + serverCfg.getInetAddress(), + serverCfg.getSslCertCfg().getCertFingerPrintStr()); sslEngine = sslContext.createSSLEngine(); sslEngine.setUseClientMode(true); } diff --git a/src/java/main/org/apache/zookeeper/Login.java b/src/java/main/org/apache/zookeeper/Login.java index fd9a4c24672..26219e39159 100644 --- a/src/java/main/org/apache/zookeeper/Login.java +++ b/src/java/main/org/apache/zookeeper/Login.java @@ -25,27 +25,26 @@ * See ZooKeeperSaslClient for client-side usage. */ +import java.util.Date; +import java.util.Random; +import java.util.Set; + +import javax.security.auth.Subject; +import javax.security.auth.callback.CallbackHandler; import javax.security.auth.kerberos.KerberosPrincipal; +import javax.security.auth.kerberos.KerberosTicket; import javax.security.auth.login.AppConfigurationEntry; import javax.security.auth.login.Configuration; import javax.security.auth.login.LoginContext; import javax.security.auth.login.LoginException; -import javax.security.auth.callback.CallbackHandler; import org.apache.zookeeper.client.ZKClientConfig; +import org.apache.zookeeper.common.Time; import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.server.ZooKeeperSaslServer; -import org.apache.zookeeper.common.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.security.auth.kerberos.KerberosTicket; -import javax.security.auth.Subject; - -import java.util.Date; -import java.util.Random; -import java.util.Set; - public class Login { private static final String KINIT_COMMAND_DEFAULT = "/usr/bin/kinit"; private static final Logger LOG = LoggerFactory.getLogger(Login.class); @@ -206,7 +205,7 @@ public void run() { break; } if (isUsingTicketCache) { - String cmd = zkConfig.getProperty(ZKConfig.KINIT_COMMAND, KINIT_COMMAND_DEFAULT); + String cmd = zkConfig.getProperty(ZKClientConfig.KINIT_COMMAND, KINIT_COMMAND_DEFAULT); String kinitArgs = "-R"; int retry = 1; while (retry >= 0) { diff --git a/src/java/main/org/apache/zookeeper/SSLCertCfg.java b/src/java/main/org/apache/zookeeper/SSLCertCfg.java index 71db5757308..77027cf1244 100644 --- a/src/java/main/org/apache/zookeeper/SSLCertCfg.java +++ b/src/java/main/org/apache/zookeeper/SSLCertCfg.java @@ -23,7 +23,6 @@ import java.util.Map; import org.apache.zookeeper.common.X509Util; -import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -119,7 +118,7 @@ private static MessageDigest getMessageDigest(final String fp) // Check for supported algos for the given fingerprint if cannot // validate throw exception. MessageDigest md = X509Util.getSupportedMessageDigestForFpStr( - new ZKConfig(), fp); + new QuorumPeerConfig(), fp); if (md == null) { final String errStr = "Algo in fingerprint: " + fp + " not supported, bailing out"; diff --git a/src/java/main/org/apache/zookeeper/client/ClientX509Util.java b/src/java/main/org/apache/zookeeper/client/ClientX509Util.java new file mode 100644 index 00000000000..bc2edc72c6e --- /dev/null +++ b/src/java/main/org/apache/zookeeper/client/ClientX509Util.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.client; + +import java.net.InetSocketAddress; + +import javax.net.ssl.SSLContext; + +import org.apache.zookeeper.common.X509ChainedTrustManager; +import org.apache.zookeeper.common.X509Exception; +import org.apache.zookeeper.common.X509Util; +import org.apache.zookeeper.common.ZKConfig; +import org.apache.zookeeper.server.quorum.util.ZKPeerX509TrustManager; +import org.apache.zookeeper.server.quorum.util.ZKX509TrustManager; + +public class ClientX509Util extends X509Util { + /** + * SSL context which depends given peer's digest along with regular + * verification via Truststore done first. + * @param zkConfig Given client config. + * @param peerAddr The Zookeeper server address to connect to + * @param peerCertFingerPrintStr The digest for the above server. + * @return SSLContext which can perform authentication. + * @throws X509Exception.SSLContextException + */ + public static SSLContext createSSLContext( + final ZKConfig zkConfig, final InetSocketAddress peerAddr, + final String peerCertFingerPrintStr) + throws X509Exception.SSLContextException { + try { + return createSSLContext(zkConfig, new X509ChainedTrustManager( + new ZKX509TrustManager(zkConfig.getProperty( + ZKConfig.SSL_TRUSTSTORE_LOCATION), + zkConfig.getProperty( + ZKConfig.SSL_TRUSTSTORE_PASSWD)), + new ZKPeerX509TrustManager(zkConfig, + peerAddr, peerCertFingerPrintStr))); + } catch (X509Exception.TrustManagerException exp) { + throw new X509Exception.SSLContextException(exp); + } + } +} diff --git a/src/java/main/org/apache/zookeeper/client/FourLetterWordMain.java b/src/java/main/org/apache/zookeeper/client/FourLetterWordMain.java index 7a19020ab1b..219fa1c9958 100644 --- a/src/java/main/org/apache/zookeeper/client/FourLetterWordMain.java +++ b/src/java/main/org/apache/zookeeper/client/FourLetterWordMain.java @@ -32,8 +32,6 @@ import javax.net.ssl.SSLSocketFactory; import org.apache.zookeeper.common.X509Exception.SSLContextException; -import org.apache.zookeeper.common.X509Util; -import org.apache.zookeeper.common.ZKConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,8 +89,8 @@ public static String send4LetterWord(String host, int port, String cmd, boolean LOG.info("using secure socket"); // TODO: 4 letter main cannot be secure!, not supported yet. // Which means this has to be local only. - SSLContext sslContext = X509Util.createSSLContext( - new ZKConfig(), null); + SSLContext sslContext = ClientX509Util.createSSLContext( + new ZKClientConfig(), hostaddress, "invalid-digest"); SSLSocketFactory socketFactory = sslContext.getSocketFactory(); SSLSocket sslSock = (SSLSocket) socketFactory.createSocket(); sslSock.connect(hostaddress, timeout); diff --git a/src/java/main/org/apache/zookeeper/client/ZKClientConfig.java b/src/java/main/org/apache/zookeeper/client/ZKClientConfig.java index 0eab9c5be17..cae6dfa8878 100644 --- a/src/java/main/org/apache/zookeeper/client/ZKClientConfig.java +++ b/src/java/main/org/apache/zookeeper/client/ZKClientConfig.java @@ -21,14 +21,14 @@ import java.io.File; import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.common.ZKConfig; +import org.apache.zookeeper.server.ZookeeperServerConfig; import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException; /** * Handles client specific properties * @since 3.5.2 */ -public class ZKClientConfig extends ZKConfig { +public class ZKClientConfig extends ZookeeperServerConfig { public static final String ZK_SASL_CLIENT_USERNAME = "zookeeper.sasl.client.username"; public static final String ZK_SASL_CLIENT_USERNAME_DEFAULT = "zookeeper"; @SuppressWarnings("deprecation") diff --git a/src/java/main/org/apache/zookeeper/client/ZooKeeperSaslClient.java b/src/java/main/org/apache/zookeeper/client/ZooKeeperSaslClient.java index 449e7be6a9c..626e741c7fc 100644 --- a/src/java/main/org/apache/zookeeper/client/ZooKeeperSaslClient.java +++ b/src/java/main/org/apache/zookeeper/client/ZooKeeperSaslClient.java @@ -43,7 +43,6 @@ import org.apache.zookeeper.Login; import org.apache.zookeeper.Watcher.Event.KeeperState; import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.proto.GetSASLRequest; import org.apache.zookeeper.proto.SetSASLResponse; @@ -280,7 +279,7 @@ private SaslClient createSaslClient(final String servicePrincipal, final String return saslClient; } else { // GSSAPI. - boolean usingNativeJgss = clientConfig.getBoolean(ZKConfig.JGSS_NATIVE); + boolean usingNativeJgss = clientConfig.getBoolean(ZKClientConfig.JGSS_NATIVE); if (usingNativeJgss) { // http://docs.oracle.com/javase/6/docs/technotes/guides/security/jgss/jgss-features.html // """ diff --git a/src/java/main/org/apache/zookeeper/common/SslConfig.java b/src/java/main/org/apache/zookeeper/common/SslConfig.java new file mode 100644 index 00000000000..64050eaa255 --- /dev/null +++ b/src/java/main/org/apache/zookeeper/common/SslConfig.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.common; + + +public abstract class SslConfig { + public String getSslVersionDefault() { + return ZKConfig.SSL_VERSION_DEFAULT; + } + + public String getSslVersion() { + return getKey(ZKConfig.SSL_VERSION); + } + + public String getSslKeyStoreLocation() { + return getKey(ZKConfig.SSL_KEYSTORE_LOCATION); + } + + public String getSslKeyStorePassword() { + return getKey(ZKConfig.SSL_KEYSTORE_PASSWD); + } + + public String getSslTrustStoreLocation() { + return getKey(ZKConfig.SSL_TRUSTSTORE_LOCATION); + } + + public String getSslTrustStorePassword() { + return getKey(ZKConfig.SSL_TRUSTSTORE_PASSWD); + } + + public String getSslAuthProvider() { + return getKey(ZKConfig.SSL_AUTHPROVIDER); + } + + public String getSslDefaultDigestAlgo() { + return ZKConfig.SSL_DIGEST_DEFAULT_ALGO; + } + + public String getSslDigestAlgos() { + return getKey(ZKConfig.SSL_DIGEST_ALGOS); + } + + protected abstract String getPrefix(); + + private String getKey(final String suffix) { + return getPrefix() + "." + suffix; + } +} diff --git a/src/java/main/org/apache/zookeeper/common/X509ChainedTrustManager.java b/src/java/main/org/apache/zookeeper/common/X509ChainedTrustManager.java new file mode 100644 index 00000000000..00ba25e28ff --- /dev/null +++ b/src/java/main/org/apache/zookeeper/common/X509ChainedTrustManager.java @@ -0,0 +1,218 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.common; + +import java.net.Socket; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import javax.net.ssl.SSLEngine; +import javax.net.ssl.X509ExtendedTrustManager; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class X509ChainedTrustManager extends X509ExtendedTrustManager { + private static final Logger LOG = LoggerFactory.getLogger( + X509ChainedTrustManager.class); + + public class TrustManagerWithPredicate { + final X509ExtendedTrustManager trustManager; + final Boolean predicate; + public TrustManagerWithPredicate( + final X509ExtendedTrustManager trustManager, + final Boolean predicate) { + this.trustManager = trustManager; + this.predicate = predicate; + } + + public X509ExtendedTrustManager getTrustManager() { + return predicate ? trustManager : null; + } + } + + private final List trustManagerPredicateList; + + public X509ChainedTrustManager( + final X509ExtendedTrustManager ... trustManagers) { + final List trustManagerWithPredicateList = + new ArrayList<>(); + for (final X509ExtendedTrustManager trustManager : trustManagers) { + trustManagerWithPredicateList.add( + new TrustManagerWithPredicate(trustManager, true)); + } + this.trustManagerPredicateList = + Collections.unmodifiableList(trustManagerWithPredicateList); + } + + public X509ChainedTrustManager( + final TrustManagerWithPredicate ... trustManagerWithPredicates) { + trustManagerPredicateList = Arrays.asList(trustManagerWithPredicates); + } + + @Override + public void checkClientTrusted(final X509Certificate[] x509Certificates, + final String s) throws CertificateException { + final List expErrMsgList = new ArrayList<>(); + for (final TrustManagerWithPredicate trustManagerWithPredicate : + trustManagerPredicateList) { + if (trustManagerWithPredicate.getTrustManager() != null) { + try { + trustManagerWithPredicate.getTrustManager() + .checkClientTrusted(x509Certificates, s); + return; // means success, i.e no exception raised. + } catch(CertificateException exp) { + // Ignore each exception. + expErrMsgList.add(exp.getMessage()); + } + } + } + final String errStr = "Could not verify certificate chain: " + + String.join(" ", expErrMsgList); + LOG.error(errStr); + throw new CertificateException(errStr); + } + + @Override + public void checkServerTrusted(final X509Certificate[] x509Certificates, + final String s) throws CertificateException { + final List expErrMsgList = new ArrayList<>(); + for (final TrustManagerWithPredicate trustManagerWithPredicate : + trustManagerPredicateList) { + if (trustManagerWithPredicate.getTrustManager() != null) { + try { + trustManagerWithPredicate.getTrustManager() + .checkServerTrusted(x509Certificates, s); + return; // means success, i.e no exception raised. + } catch(CertificateException exp) { + // Ignore each exception. + expErrMsgList.add(exp.getMessage()); + } + } + } + final String errStr = "Could not verify certificate chain: " + + String.join(" ", expErrMsgList); + LOG.error(errStr); + throw new CertificateException(errStr); + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + + @Override + public void checkClientTrusted(final X509Certificate[] x509Certificates, + final String s, final Socket socket) + throws CertificateException { + final List expErrMsgList = new ArrayList<>(); + for (final TrustManagerWithPredicate trustManagerWithPredicate : + trustManagerPredicateList) { + if (trustManagerWithPredicate.getTrustManager() != null) { + try { + trustManagerWithPredicate.getTrustManager() + .checkClientTrusted(x509Certificates, s, socket); + return; // means success, i.e no exception raised. + } catch(CertificateException exp) { + // Ignore each exception. + expErrMsgList.add(exp.getMessage()); + } + } + } + final String errStr = "Could not verify certificate chain: " + + String.join(" ", expErrMsgList); + LOG.error(errStr); + throw new CertificateException(errStr); + } + + @Override + public void checkServerTrusted(final X509Certificate[] x509Certificates, + final String s, final Socket socket) + throws CertificateException { + final List expErrMsgList = new ArrayList<>(); + for (final TrustManagerWithPredicate trustManagerWithPredicate : + trustManagerPredicateList) { + if (trustManagerWithPredicate.getTrustManager() != null) { + try { + trustManagerWithPredicate.getTrustManager() + .checkServerTrusted(x509Certificates, s, socket); + return; // means success, i.e no exception raised. + } catch(CertificateException exp) { + // Ignore each exception. + expErrMsgList.add(exp.getMessage()); + } + } + } + final String errStr = "Could not verify certificate chain: " + + String.join(" ", expErrMsgList); + LOG.error(errStr); + throw new CertificateException(errStr); + } + + @Override + public void checkClientTrusted(final X509Certificate[] x509Certificates, + final String s, final SSLEngine sslEngine) + throws CertificateException { + final List expErrMsgList = new ArrayList<>(); + for (final TrustManagerWithPredicate trustManagerWithPredicate : + trustManagerPredicateList) { + if (trustManagerWithPredicate.getTrustManager() != null) { + try { + trustManagerWithPredicate.getTrustManager() + .checkClientTrusted(x509Certificates, s, sslEngine); + return; // means success, i.e no exception raised. + } catch(CertificateException exp) { + // Ignore each exception. + expErrMsgList.add(exp.getMessage()); + } + } + } + final String errStr = "Could not verify certificate chain: " + + String.join(" ", expErrMsgList); + LOG.error(errStr); + throw new CertificateException(errStr); + } + + @Override + public void checkServerTrusted(final X509Certificate[] x509Certificates, + final String s, final SSLEngine sslEngine) + throws CertificateException { + final List expErrMsgList = new ArrayList<>(); + for (final TrustManagerWithPredicate trustManagerWithPredicate : + trustManagerPredicateList) { + if (trustManagerWithPredicate.getTrustManager() != null) { + try { + trustManagerWithPredicate.getTrustManager() + .checkServerTrusted(x509Certificates, s, sslEngine); + return; // means success, i.e no exception raised. + } catch(CertificateException exp) { + // Ignore each exception. + expErrMsgList.add(exp.getMessage()); + } + } + } + final String errStr = "Could not verify certificate chain: " + + String.join(" ", expErrMsgList); + LOG.error(errStr); + throw new CertificateException(errStr); + } +} diff --git a/src/java/main/org/apache/zookeeper/common/X509Util.java b/src/java/main/org/apache/zookeeper/common/X509Util.java index 637a05e6afd..6d705db19a0 100644 --- a/src/java/main/org/apache/zookeeper/common/X509Util.java +++ b/src/java/main/org/apache/zookeeper/common/X509Util.java @@ -39,6 +39,7 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509ExtendedTrustManager; import javax.net.ssl.X509KeyManager; import javax.net.ssl.X509TrustManager; import javax.xml.bind.DatatypeConverter; @@ -60,120 +61,19 @@ * Utility code for X509 handling */ public class X509Util { - private static final Logger LOG = LoggerFactory.getLogger(X509Util.class); + protected static final Logger LOG = LoggerFactory.getLogger(X509Util.class); - /** - * @deprecated Use {@link ZKConfig#SSL_VERSION_DEFAULT} - * instead. - */ - @Deprecated - public static final String SSL_VERSION_DEFAULT = "TLSv1"; - /** - * @deprecated Use {@link ZKConfig#SSL_VERSION} - * instead. - */ - @Deprecated - public static final String SSL_VERSION = "zookeeper.ssl.version"; - /** - * @deprecated Use {@link ZKConfig#SSL_KEYSTORE_LOCATION} - * instead. - */ - @Deprecated - public static final String SSL_KEYSTORE_LOCATION = "zookeeper.ssl.keyStore.location"; - /** - * @deprecated Use {@link ZKConfig#SSL_KEYSTORE_PASSWD} - * instead. - */ - @Deprecated - public static final String SSL_KEYSTORE_PASSWD = "zookeeper.ssl.keyStore.password"; - /** - * @deprecated Use {@link ZKConfig#SSL_TRUSTSTORE_LOCATION} - * instead. - */ - @Deprecated - public static final String SSL_TRUSTSTORE_LOCATION = "zookeeper.ssl.trustStore.location"; - /** - * @deprecated Use {@link ZKConfig#SSL_TRUSTSTORE_PASSWD} - * instead. - */ - @Deprecated - public static final String SSL_TRUSTSTORE_PASSWD = "zookeeper.ssl.trustStore.password"; - /** - * @deprecated Use {@link ZKConfig#SSL_AUTHPROVIDER} - * instead. - */ - @Deprecated - public static final String SSL_AUTHPROVIDER = "zookeeper.ssl.authProvider"; - - /** - * @deprecated Use {@link ZKConfig#SSL_TRUSTSTORE_CA_ALIAS} - * instead. - */ - @Deprecated - public static final String SSL_TRUSTSTORE_CA_ALIAS = - "zookeeper.ssl.trustStore.rootCA.alias"; - /** - * @deprecated Use {@link ZKConfig#SSL_DIGEST_DEFAULT_ALGO} - * instead. - */ - @Deprecated - public static final String SSL_DIGEST_DEFAULT_ALGO ="SHA-256"; - /** - * @deprecated Use {@link ZKConfig#SSL_DIGEST_ALGOS} - * instead. - */ - @Deprecated - public static final String SSL_DIGEST_ALGOS = "quorum.ssl.digest.algos"; - - /** - * API to create SSL context for clients. Supports both self-signed and - * CA signed. - * @param peerAddr host that client is trying to connect to - * @param peerCertCfg host's self-signed cert of CA signed cert. - * @return SSLContext with right verification based on self-signed or CA - * signed. - * @throws SSLContextException - */ public static SSLContext createSSLContext( final ZKConfig config, - final InetSocketAddress peerAddr, - final SSLCertCfg peerCertCfg) - throws SSLContextException { + final X509ExtendedTrustManager trustManager) + throws SSLContextException { final KeyManager[] keyManagers = createKeyManagers(config); - TrustManager[] trustManagers; - if (peerCertCfg.isSelfSigned()) { - trustManagers = createTrustManagers(config, peerAddr, - peerCertCfg.getCertFingerPrintStr()); - } else if (peerCertCfg.isCASigned()) { - // Lets load the CA for truststore. - trustManagers = createTrustManagers(config, null); - } else { - throw new IllegalArgumentException("Invalid argument, no SSL cfg " + - "provided"); - } - - return createSSLContext(config, keyManagers, trustManagers); + return createSSLContext(config, keyManagers, + new TrustManager[]{trustManager}); } - /** - * SSL context which can be used by both client and server side which - * depend on dynamic config for authentication. Hence we need quorumPeer. - * @param quorumPeer Used for getting QuorumVerifier and certs from - * QuorumPeerConfig. Both commited and last verified. - * @return SSLContext which can perform authentication based on dynamic cfg. - * @throws SSLContextException - */ - public static SSLContext createSSLContext(final ZKConfig config, - final QuorumPeer quorumPeer) - throws SSLContextException { - final KeyManager[] keyManagers = createKeyManagers(config); - final TrustManager[] trustManagers = - createTrustManagers(config, quorumPeer); - - return createSSLContext(config, keyManagers, trustManagers); - } - - private static KeyManager[] createKeyManagers(final ZKConfig config) throws + protected static KeyManager[] createKeyManagers(final ZKConfig config) + throws SSLContextException { final String keyStoreLocationProp = config.getProperty(ZKConfig.SSL_KEYSTORE_LOCATION); @@ -202,60 +102,8 @@ private static KeyManager[] createKeyManagers(final ZKConfig config) throws } } - /** - * If QuorumPeer is not provided and this is called it means we are CA - * mode and need both truststore location and password. - * @param quorumPeer - * @return - * @throws SSLContextException - */ - private static TrustManager[] createTrustManagers( - final ZKConfig config, - final QuorumPeer quorumPeer) throws SSLContextException { - String trustStoreLocationProp = - config.getProperty(ZKConfig.SSL_TRUSTSTORE_LOCATION); - String trustStorePasswordProp = - config.getProperty(ZKConfig.SSL_TRUSTSTORE_PASSWD); - - if (trustStoreLocationProp == null && trustStorePasswordProp == null) { - if (quorumPeer == null) { - final String errStr = "truststore not specified"; - LOG.error(errStr); - throw new SSLContextException(errStr); - } - - // Create self-signed verification using QuorumPeer. - return new TrustManager[] {createTrustManager(quorumPeer)}; - } else { - if (trustStoreLocationProp == null) { - throw new SSLContextException("truststore location not " + - "specified for client connection"); - } - if (trustStorePasswordProp == null) { - throw new SSLContextException("truststore password not " + - "specified for client connection"); - } - try { - return new TrustManager[] { - createTrustManager(config, trustStoreLocationProp, - trustStorePasswordProp)}; - } catch (TrustManagerException e) { - throw new SSLContextException( - "Failed to create TrustManager", e); - } - } - } - - private static TrustManager[] createTrustManagers( - final ZKConfig config, - final InetSocketAddress peerAddr, - final String peerCertFingerPrintStr) { - return new TrustManager[]{ - createTrustManager(config, peerAddr, - peerCertFingerPrintStr)}; - } - private static SSLContext createSSLContext( + protected static SSLContext createSSLContext( final ZKConfig config, final KeyManager[] keyManagers, final TrustManager[] trustManagers) @@ -297,13 +145,13 @@ public static X509KeyManager createKeyManager( } } - private static KeyStore loadKeyStore(final String keyStoreLocation, - final String keyStorePassword) - throws KeyStoreException, NoSuchAlgorithmException, + public static KeyStore loadKeyStore(final String keyStoreLocation, + final String keyStorePassword) + throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException { - char[] keyStorePasswordChars = keyStorePassword.toCharArray(); - File keyStoreFile = new File(keyStoreLocation); - KeyStore ks = KeyStore.getInstance("JKS"); + final char[] keyStorePasswordChars = keyStorePassword.toCharArray(); + final File keyStoreFile = new File(keyStoreLocation); + final KeyStore ks = KeyStore.getInstance("JKS"); try (final FileInputStream inputStream = new FileInputStream (keyStoreFile)) { ks.load(inputStream, keyStorePasswordChars); @@ -311,63 +159,6 @@ private static KeyStore loadKeyStore(final String keyStoreLocation, return ks; } - private static X509TrustManager createTrustManager( - final ZKConfig config, - final InetSocketAddress peerAddr, - final String peerCertFingerPrintStr) { - return new ZKPeerX509TrustManager(peerAddr, peerCertFingerPrintStr); - } - - public static X509TrustManager createTrustManager( - final ZKConfig config, - final String trustStoreLocation, final String trustStorePassword) - throws TrustManagerException, SSLContextException { - String trustStoreCAAlias = - config.getProperty(ZKConfig.SSL_TRUSTSTORE_CA_ALIAS); - if (trustStoreCAAlias == null) { - final String errStr = "No CA Alias provided, need one to work in " + - "CA mode."; - LOG.error(errStr); - throw new TrustManagerException(errStr); - } - try(final FileInputStream inputStream = - new FileInputStream(new File(trustStoreLocation))) { - char[] trustStorePasswordChars = - trustStorePassword.toCharArray(); - KeyStore ts = KeyStore.getInstance("JKS"); - ts.load(inputStream, trustStorePasswordChars); - TrustManagerFactory tmf = - TrustManagerFactory.getInstance("SunX509"); - tmf.init(ts); - X509Certificate rootCA = - getCertWithAlias(ts, trustStoreCAAlias); - if (rootCA == null) { - final String str = "failed to find root CA from: " + - trustStoreLocation + " with alias: " + - trustStoreCAAlias; - LOG.error(str); - throw new TrustManagerException(str); - } - - return createTrustManager(rootCA); - } catch (IOException | KeyStoreException | NoSuchAlgorithmException - | CertificateException e) { - final String errStr = "Could not load truststore: " - + trustStoreLocation; - LOG.error("{}", errStr, e); - throw new TrustManagerException(errStr, e); - } - } - - private static X509TrustManager createTrustManager( - final X509Certificate rootCA) { - return new ZKX509TrustManager(rootCA); - } - - public static X509TrustManager createTrustManager( - final QuorumPeer quorumPeer) { - return new ZKDynamicX509TrustManager(quorumPeer); - } private static X509Certificate getCertWithAlias( final KeyStore trustStore, final String alias) throws KeyStoreException { diff --git a/src/java/main/org/apache/zookeeper/common/ZKConfig.java b/src/java/main/org/apache/zookeeper/common/ZKConfig.java index c3ff34fd87b..3ab5009334c 100644 --- a/src/java/main/org/apache/zookeeper/common/ZKConfig.java +++ b/src/java/main/org/apache/zookeeper/common/ZKConfig.java @@ -28,6 +28,7 @@ import org.apache.zookeeper.Environment; import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException; +import org.apache.zookeeper.server.quorum.SyncedLearnerTracker; import org.apache.zookeeper.server.util.VerifyingFileFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,38 +40,22 @@ * {@link #setProperty(String, String)}. * @since 3.5.2 */ -public class ZKConfig { - +public abstract class ZKConfig { private static final Logger LOG = LoggerFactory.getLogger(ZKConfig.class); - @SuppressWarnings("deprecation") - public static final String SSL_VERSION_DEFAULT = X509Util.SSL_VERSION_DEFAULT; - @SuppressWarnings("deprecation") - public static final String SSL_VERSION = X509Util.SSL_VERSION; - @SuppressWarnings("deprecation") - public static final String SSL_KEYSTORE_LOCATION = X509Util.SSL_KEYSTORE_LOCATION; - @SuppressWarnings("deprecation") - public static final String SSL_KEYSTORE_PASSWD = X509Util.SSL_KEYSTORE_PASSWD; - @SuppressWarnings("deprecation") - public static final String SSL_TRUSTSTORE_LOCATION = X509Util.SSL_TRUSTSTORE_LOCATION; - @SuppressWarnings("deprecation") - public static final String SSL_TRUSTSTORE_PASSWD = X509Util.SSL_TRUSTSTORE_PASSWD; - @SuppressWarnings("deprecation") - public static final String SSL_AUTHPROVIDER = X509Util.SSL_AUTHPROVIDER; - @SuppressWarnings("deprecation") - public static final String SSL_TRUSTSTORE_CA_ALIAS = X509Util.SSL_TRUSTSTORE_CA_ALIAS; - @SuppressWarnings("deprecation") - public static final String SSL_DIGEST_DEFAULT_ALGO = X509Util.SSL_DIGEST_DEFAULT_ALGO; - @SuppressWarnings("deprecation") - public static final String SSL_DIGEST_ALGOS = X509Util.SSL_DIGEST_ALGOS; - public static final String JUTE_MAXBUFFER = "jute.maxbuffer"; - /** - * Path to a kinit binary: {@value}. Defaults to - * "/usr/bin/kinit" - */ - public static final String KINIT_COMMAND = "zookeeper.kinit"; - public static final String JGSS_NATIVE = "sun.security.jgss.native"; - private final Map properties = new HashMap(); + public static final String SSL_VERSION_DEFAULT = "TLSv1"; + public static final String SSL_VERSION = "ssl.version"; + public static final String SSL_KEYSTORE_LOCATION = "ssl.keyStore"; + public static final String SSL_KEYSTORE_PASSWD = "ssl.keyStore.password"; + public static final String SSL_TRUSTSTORE_LOCATION = + "ssl.trustStore.location"; + public static final String SSL_TRUSTSTORE_PASSWD = + "ssl.trustStore.password"; + public static final String SSL_AUTHPROVIDER = "ssl.authProvider"; + public static final String SSL_DIGEST_DEFAULT_ALGO = "SHA-256"; + public static final String SSL_DIGEST_ALGOS = "ssl.digest.algos"; + + protected final Map properties = new HashMap(); /** * properties, which are common to both client and server, are initialized @@ -117,17 +102,30 @@ private void init() { * this configuration. */ protected void handleBackwardCompatibility() { - properties.put(SSL_VERSION, System.getProperty(SSL_VERSION)); - properties.put(SSL_KEYSTORE_LOCATION, System.getProperty(SSL_KEYSTORE_LOCATION)); - properties.put(SSL_KEYSTORE_PASSWD, System.getProperty(SSL_KEYSTORE_PASSWD)); - properties.put(SSL_TRUSTSTORE_LOCATION, System.getProperty(SSL_TRUSTSTORE_LOCATION)); - properties.put(SSL_TRUSTSTORE_PASSWD, System.getProperty(SSL_TRUSTSTORE_PASSWD)); - properties.put(SSL_AUTHPROVIDER, System.getProperty(SSL_AUTHPROVIDER)); - properties.put(SSL_TRUSTSTORE_CA_ALIAS, System.getProperty(SSL_TRUSTSTORE_CA_ALIAS)); - properties.put(SSL_DIGEST_ALGOS, System.getProperty(SSL_DIGEST_ALGOS)); - properties.put(JUTE_MAXBUFFER, System.getProperty(JUTE_MAXBUFFER)); - properties.put(KINIT_COMMAND, System.getProperty(KINIT_COMMAND)); - properties.put(JGSS_NATIVE, System.getProperty(JGSS_NATIVE)); + properties.put(SSL_VERSION, systemPropDefault( + getSslConfig().getSslVersion(), + getSslConfig().getSslVersionDefault())); + properties.put(SSL_KEYSTORE_LOCATION, + System.getProperty(getSslConfig().getSslKeyStoreLocation())); + properties.put(SSL_KEYSTORE_PASSWD, + System.getProperty(getSslConfig().getSslKeyStorePassword())); + properties.put(SSL_TRUSTSTORE_LOCATION, + System.getProperty(getSslConfig().getSslTrustStoreLocation())); + properties.put(SSL_TRUSTSTORE_PASSWD, + System.getProperty(getSslConfig().getSslTrustStorePassword())); + properties.put(SSL_AUTHPROVIDER, + getSslConfig().getSslAuthProvider()); + properties.put(SSL_DIGEST_ALGOS, systemPropDefault( + getSslConfig().getSslDigestAlgos(), + getSslConfig().getSslDefaultDigestAlgo())); + } + + protected abstract SslConfig getSslConfig(); + + protected static String systemPropDefault(final String key, + final String defaultValue) { + return System.getProperty(key) != null ? System.getProperty(key) : + defaultValue; } /** diff --git a/src/java/main/org/apache/zookeeper/server/NettyServerCnxnFactory.java b/src/java/main/org/apache/zookeeper/server/NettyServerCnxnFactory.java index bec7c26bae5..199512c3a84 100644 --- a/src/java/main/org/apache/zookeeper/server/NettyServerCnxnFactory.java +++ b/src/java/main/org/apache/zookeeper/server/NettyServerCnxnFactory.java @@ -33,12 +33,15 @@ import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLPeerUnverifiedException; import javax.net.ssl.SSLSession; +import javax.net.ssl.X509KeyManager; +import javax.net.ssl.X509TrustManager; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.common.X509Exception; -import org.apache.zookeeper.common.X509Util; import org.apache.zookeeper.common.ZKConfig; +import org.apache.zookeeper.server.auth.ProviderRegistry; import org.apache.zookeeper.server.auth.X509AuthenticationProvider; +import org.apache.zookeeper.server.util.ServerX509Util; import org.jboss.netty.bootstrap.ServerBootstrap; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBuffers; @@ -338,24 +341,23 @@ public ChannelPipeline getPipeline() throws Exception { private synchronized void initSSL(ChannelPipeline p) throws X509Exception, KeyManagementException, NoSuchAlgorithmException { - String authProviderProp = System.getProperty(ZKConfig.SSL_AUTHPROVIDER); + final ZKConfig zkConfig = new ZookeeperServerConfig(); + String authProviderProp = zkConfig.getProperty(ZKConfig.SSL_AUTHPROVIDER); SSLContext sslContext; if (authProviderProp == null) { - sslContext = X509Util.createSSLContext(new ZKConfig(), quorumPeer); + sslContext = ServerX509Util.createSSLContext(quorumPeer); } else { - throw new IllegalAccessError("No support for auth provider: " + - authProviderProp); - /* sslContext = SSLContext.getInstance("TLSv1"); X509AuthenticationProvider authProvider = - (X509AuthenticationProvider)ProviderRegistry.getProvider( - System.getProperty(ZKConfig.SSL_AUTHPROVIDER, + (X509AuthenticationProvider) ProviderRegistry.getProvider( + System.getProperty( + new ZookeeperServerSslConfig().getSslAuthProvider(), "x509")); if (authProvider == null) { LOG.error("Auth provider not found: {}", authProviderProp); - throw new SSLContextException( + throw new X509Exception.SSLContextException( "Could not create SSLContext with specified auth provider: " + authProviderProp); } @@ -363,7 +365,6 @@ private synchronized void initSSL(ChannelPipeline p) sslContext.init(new X509KeyManager[] { authProvider.getKeyManager() }, new X509TrustManager[] { authProvider.getTrustManager() }, null); - */ } SSLEngine sslEngine = sslContext.createSSLEngine(); diff --git a/src/java/main/org/apache/zookeeper/server/ServerCnxnFactory.java b/src/java/main/org/apache/zookeeper/server/ServerCnxnFactory.java index 1f5053b7898..cd3889551a6 100644 --- a/src/java/main/org/apache/zookeeper/server/ServerCnxnFactory.java +++ b/src/java/main/org/apache/zookeeper/server/ServerCnxnFactory.java @@ -241,7 +241,7 @@ protected void configureSaslLogin() throws IOException { // jaas.conf entry available try { saslServerCallbackHandler = new SaslServerCallbackHandler(Configuration.getConfiguration()); - login = new Login(serverSection, saslServerCallbackHandler, new ZKConfig() ); + login = new Login(serverSection, saslServerCallbackHandler, new ZookeeperServerConfig()); login.startThreadIfNeeded(); } catch (LoginException e) { throw new IOException("Could not configure server because SASL configuration did not allow the " diff --git a/src/java/main/org/apache/zookeeper/server/ZookeeperServerConfig.java b/src/java/main/org/apache/zookeeper/server/ZookeeperServerConfig.java new file mode 100644 index 00000000000..935e83190c2 --- /dev/null +++ b/src/java/main/org/apache/zookeeper/server/ZookeeperServerConfig.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.server; + +import java.io.File; + +import org.apache.zookeeper.common.SslConfig; +import org.apache.zookeeper.common.ZKConfig; +import org.apache.zookeeper.server.quorum.QuorumPeerConfig; + +public class ZookeeperServerConfig extends ZKConfig { + public static final String JUTE_MAXBUFFER = "jute.maxbuffer"; + /** + * Path to a kinit binary: {@value}. Defaults to + * "/usr/bin/kinit" + */ + public static final String KINIT_COMMAND = "zookeeper.kinit"; + public static final String JGSS_NATIVE = "sun.security.jgss.native"; + private final ZookeeperServerSslConfig sslConfig = + new ZookeeperServerSslConfig(); + + public ZookeeperServerConfig() { + super(); + } + + public ZookeeperServerConfig(File configFile) + throws QuorumPeerConfig.ConfigException { + super(configFile); + } + + public ZookeeperServerConfig(String configPath) + throws QuorumPeerConfig.ConfigException { + super(configPath); + } + + /** + * Now onwards client code will use properties from this class but older + * clients still be setting properties through system properties. So to make + * this change backward compatible we should set old system properties in + * this configuration. + */ + @Override + protected void handleBackwardCompatibility() { + super.handleBackwardCompatibility(); + properties.put(JUTE_MAXBUFFER, System.getProperty(JUTE_MAXBUFFER)); + properties.put(KINIT_COMMAND, System.getProperty(KINIT_COMMAND)); + properties.put(JGSS_NATIVE, System.getProperty(JGSS_NATIVE)); + } + + @Override + protected SslConfig getSslConfig() { + return sslConfig; + } +} diff --git a/src/java/main/org/apache/zookeeper/server/ZookeeperServerSslConfig.java b/src/java/main/org/apache/zookeeper/server/ZookeeperServerSslConfig.java new file mode 100644 index 00000000000..3e975925e53 --- /dev/null +++ b/src/java/main/org/apache/zookeeper/server/ZookeeperServerSslConfig.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.server; + + +import org.apache.zookeeper.common.SslConfig; + +public class ZookeeperServerSslConfig extends SslConfig { + private static final String PREFIX = "zookeeper"; + + public static final String SSL_AUTHPROVIDER = "zookeeper.ssl.authProvider"; + + public String getSslAuthProvider() { + return SSL_AUTHPROVIDER; + } + + @Override + protected String getPrefix() { + return PREFIX; + } +} diff --git a/src/java/main/org/apache/zookeeper/server/auth/X509AuthenticationProvider.java b/src/java/main/org/apache/zookeeper/server/auth/X509AuthenticationProvider.java index 300fe7ee51f..64c560b9779 100644 --- a/src/java/main/org/apache/zookeeper/server/auth/X509AuthenticationProvider.java +++ b/src/java/main/org/apache/zookeeper/server/auth/X509AuthenticationProvider.java @@ -26,14 +26,15 @@ import javax.security.auth.x500.X500Principal; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.common.X509Exception; import org.apache.zookeeper.common.X509Exception.KeyManagerException; import org.apache.zookeeper.common.X509Exception.TrustManagerException; import org.apache.zookeeper.common.X509Util; import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.data.Id; import org.apache.zookeeper.server.ServerCnxn; +import org.apache.zookeeper.server.ZookeeperServerConfig; import org.apache.zookeeper.server.quorum.QuorumPeer; +import org.apache.zookeeper.server.util.ServerX509Util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,8 +56,8 @@ public class X509AuthenticationProvider implements AuthenticationProvider { = "zookeeper.X509AuthenticationProvider.superUser"; private static final Logger LOG = LoggerFactory.getLogger(X509AuthenticationProvider.class); - private final X509TrustManager trustManager; private final X509KeyManager keyManager; + private final X509TrustManager trustManager; /** * Initialize the X509AuthenticationProvider with a JKS KeyStore and JKS @@ -67,39 +68,25 @@ public class X509AuthenticationProvider implements AuthenticationProvider { *
zookeeper.ssl.trustStore.password */ public X509AuthenticationProvider(final QuorumPeer quorumPeer) { - String keyStoreLocationProp = System.getProperty( - ZKConfig.SSL_KEYSTORE_LOCATION); - String keyStorePasswordProp = System.getProperty( - ZKConfig.SSL_KEYSTORE_PASSWD); - - X509KeyManager km = null; - X509TrustManager tm = null; try { - km = X509Util.createKeyManager( - keyStoreLocationProp, keyStorePasswordProp); + keyManager = X509Util.createKeyManager( + new ZookeeperServerConfig().getProperty( + ZKConfig.SSL_KEYSTORE_LOCATION), + new ZookeeperServerConfig().getProperty( + ZKConfig.SSL_KEYSTORE_PASSWD)); } catch (KeyManagerException e) { - LOG.error("Failed to create key manager", e); + final String errStr = "Failed to create key manager"; + LOG.error("{}", errStr, e); + throw new IllegalAccessError(errStr + " error: " + e.getMessage()); } - String trustStoreLocationProp = System.getProperty( - ZKConfig.SSL_TRUSTSTORE_LOCATION); - String trustStorePasswordProp = System.getProperty( - ZKConfig.SSL_TRUSTSTORE_PASSWD); - try { - if (quorumPeer != null) { - tm = X509Util.createTrustManager(quorumPeer); - } else { - tm = X509Util.createTrustManager(new ZKConfig(), - trustStoreLocationProp, trustStorePasswordProp); - } - } catch (TrustManagerException | X509Exception.SSLContextException e) { + trustManager = ServerX509Util.getTrustManager( + new ZookeeperServerConfig(), quorumPeer); + } catch (TrustManagerException e) { LOG.error("Failed to create trust manager", e); throw new IllegalAccessError("Failed to create trust manager"); } - - this.keyManager = km; - this.trustManager = tm; } /** diff --git a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java index c96e7120880..2e0e0d91e26 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java @@ -1655,7 +1655,7 @@ private String getQuorumServerFingerPrintByCert( quorumServer.getSslCertCfg().getCertFingerPrintStr(); final MessageDigest peerFpSupportedMd = X509Util.getSupportedMessageDigestForFpStr( - new ZKConfig(), peerFp); + new QuorumPeerConfig(), peerFp); if (peerFpSupportedMd == null) { final String errStr = "QuorumServer: " + quorumServer + diff --git a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java index 3dbc30ac0ee..e64348d1071 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java @@ -33,32 +33,35 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Properties; import java.util.Map.Entry; +import java.util.Properties; -import org.apache.zookeeper.common.StringUtils; -import org.apache.zookeeper.common.ZKConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.MDC; import org.apache.zookeeper.common.AtomicFileWritingIdiom; import org.apache.zookeeper.common.AtomicFileWritingIdiom.OutputStreamStatement; import org.apache.zookeeper.common.AtomicFileWritingIdiom.WriterStatement; import org.apache.zookeeper.common.PathUtils; +import org.apache.zookeeper.common.SslConfig; +import org.apache.zookeeper.common.StringUtils; +import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.server.ZooKeeperServer; +import org.apache.zookeeper.server.ZookeeperServerSslConfig; import org.apache.zookeeper.server.quorum.QuorumPeer.LearnerType; import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer; import org.apache.zookeeper.server.quorum.flexible.QuorumHierarchical; import org.apache.zookeeper.server.quorum.flexible.QuorumMaj; import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; import org.apache.zookeeper.server.util.VerifyingFileFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; -public class QuorumPeerConfig { +public class QuorumPeerConfig extends ZKConfig { private static final Logger LOG = LoggerFactory.getLogger(QuorumPeerConfig.class); private static final int UNSET_SERVERID = -1; public static final String nextDynamicConfigFileSuffix = ".dynamic.next"; + private final QuorumSslConfig sslConfig = new QuorumSslConfig(); private static boolean standaloneEnabled = true; private static boolean reconfigEnabled = false; @@ -108,6 +111,25 @@ public ConfigException(String msg, Exception e) { } } + public QuorumPeerConfig() { + super(); + } + + public QuorumPeerConfig(File configFile) + throws QuorumPeerConfig.ConfigException { + super(configFile); + } + + public QuorumPeerConfig(String configPath) + throws QuorumPeerConfig.ConfigException { + super(configPath); + } + + @Override + protected SslConfig getSslConfig() { + return sslConfig; + } + /** * Parse a ZooKeeper configuration file * @param path the patch of the configuration file @@ -375,14 +397,15 @@ public void parseProperties(Properties zkProp) * provider is not configured. */ private void configureSSLAuth() throws ConfigException { - String sslAuthProp = "zookeeper.authProvider." + System.getProperty(ZKConfig.SSL_AUTHPROVIDER, "x509"); + String sslAuthProp = "zookeeper.authProvider." + System.getProperty + (ZookeeperServerSslConfig.SSL_AUTHPROVIDER, "x509"); if (System.getProperty(sslAuthProp) == null) { if ("zookeeper.authProvider.x509".equals(sslAuthProp)) { System.setProperty("zookeeper.authProvider.x509", "org.apache.zookeeper.server.auth.X509AuthenticationProvider"); } else { throw new ConfigException("No auth provider configured for the SSL authentication scheme '" - + System.getProperty(ZKConfig.SSL_AUTHPROVIDER) + "'."); + + System.getProperty(ZookeeperServerSslConfig.SSL_AUTHPROVIDER) + "'."); } } } diff --git a/src/java/main/org/apache/zookeeper/server/quorum/QuorumSslConfig.java b/src/java/main/org/apache/zookeeper/server/quorum/QuorumSslConfig.java new file mode 100644 index 00000000000..d782ee8d4cf --- /dev/null +++ b/src/java/main/org/apache/zookeeper/server/quorum/QuorumSslConfig.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.server.quorum; + + +import org.apache.zookeeper.common.SslConfig; + +public class QuorumSslConfig extends SslConfig { + private static final String PREFIX = "quorum"; + + @Override + protected String getPrefix() { + return PREFIX; + } +} diff --git a/src/java/main/org/apache/zookeeper/server/quorum/util/QuorumSocketFactory.java b/src/java/main/org/apache/zookeeper/server/quorum/util/QuorumSocketFactory.java index e8800c6f885..38a78baa054 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/util/QuorumSocketFactory.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/util/QuorumSocketFactory.java @@ -28,8 +28,6 @@ import org.apache.zookeeper.SSLCertCfg; import org.apache.zookeeper.common.X509Exception; -import org.apache.zookeeper.common.X509Util; -import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.server.quorum.QuorumPeer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -101,9 +99,8 @@ public ServerSocket buildForServer(final QuorumPeer quorumPeer, } public Socket buildForClient(final InetSocketAddress peerAddr, - final SSLCertCfg sslCertCfg) throws - X509Exception, - IOException { + final SSLCertCfg sslCertCfg) + throws X509Exception, IOException { if (this.sslEnabled) { return newSslSocket(peerAddr, sslCertCfg); } else { @@ -120,8 +117,8 @@ private Socket newSslSocket(final InetSocketAddress peerAddr, throws X509Exception, IOException { Socket clientSocket = null; try { - clientSocket = X509Util.createSSLContext( - new ZKConfig(), peerAddr, sslCertCfg) + clientSocket = QuorumX509Util.createSSLContext(peerAddr, + sslCertCfg.getCertFingerPrintStr()) .getSocketFactory() .createSocket(); } catch (X509Exception.SSLContextException exp) { @@ -153,15 +150,15 @@ private ServerSocket newSslServerSocket( try { if (listenAddr != null) { serverSocket = - (SSLServerSocket)X509Util.createSSLContext( - new ZKConfig(), quorumPeer) + (SSLServerSocket)QuorumX509Util.createSSLContext + (quorumPeer) .getServerSocketFactory() .createServerSocket(port, backlog, listenAddr); } else { // bind to any address serverSocket = - (SSLServerSocket)X509Util.createSSLContext( - new ZKConfig(), quorumPeer) + (SSLServerSocket)QuorumX509Util.createSSLContext + (quorumPeer) .getServerSocketFactory() .createServerSocket(port, backlog); } diff --git a/src/java/main/org/apache/zookeeper/server/quorum/util/QuorumX509Util.java b/src/java/main/org/apache/zookeeper/server/quorum/util/QuorumX509Util.java new file mode 100644 index 00000000000..ee183b7052b --- /dev/null +++ b/src/java/main/org/apache/zookeeper/server/quorum/util/QuorumX509Util.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.server.quorum.util; + +import java.net.InetSocketAddress; + +import javax.net.ssl.SSLContext; + +import org.apache.zookeeper.client.ClientX509Util; +import org.apache.zookeeper.common.X509ChainedTrustManager; +import org.apache.zookeeper.common.X509Exception; +import org.apache.zookeeper.common.X509Util; +import org.apache.zookeeper.common.ZKConfig; +import org.apache.zookeeper.server.quorum.QuorumPeer; +import org.apache.zookeeper.server.quorum.QuorumPeerConfig; + +/** + * Helps with KeyManager and TrustManager creation and handling + * of certification verification for both ZAB and FLE + */ +public class QuorumX509Util extends X509Util { + /** + * SSL context which depend on dynamic config for authentication. Hence + * we need quorumPeer along with regular verification via Truststore done + * first. + * @param quorumPeer Used for getting QuorumVerifier and certs from + * QuorumPeerConfig. Both committed and last verified. + * @return SSLContext which can perform authentication based on dynamic cfg. + * @throws X509Exception.SSLContextException + */ + public static SSLContext createSSLContext(final QuorumPeer quorumPeer) + throws X509Exception.SSLContextException { + final ZKConfig zkConfig = new QuorumPeerConfig(); + try { + return createSSLContext(zkConfig, new X509ChainedTrustManager( + new ZKX509TrustManager(zkConfig.getProperty( + ZKConfig.SSL_TRUSTSTORE_LOCATION), + zkConfig.getProperty( + ZKConfig.SSL_TRUSTSTORE_PASSWD)), + new ZKDynamicX509TrustManager(quorumPeer))); + } catch (X509Exception.TrustManagerException exp) { + throw new X509Exception.SSLContextException(exp); + } + } + + public static SSLContext createSSLContext( + final InetSocketAddress peerAddr, + final String peerCertFingerPrintStr) + throws X509Exception.SSLContextException { + return ClientX509Util.createSSLContext(new QuorumPeerConfig(), + peerAddr, peerCertFingerPrintStr); + } +} diff --git a/src/java/main/org/apache/zookeeper/server/quorum/util/ZKPeerX509TrustManager.java b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKPeerX509TrustManager.java index a9c97f07bf2..0ba899b34ae 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/util/ZKPeerX509TrustManager.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKPeerX509TrustManager.java @@ -18,52 +18,105 @@ package org.apache.zookeeper.server.quorum.util; import java.net.InetSocketAddress; +import java.net.Socket; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; -import javax.net.ssl.X509TrustManager; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.X509ExtendedTrustManager; import org.apache.zookeeper.common.X509Util; import org.apache.zookeeper.common.ZKConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class ZKPeerX509TrustManager implements X509TrustManager { +/** + * TrustManager used by client to verify that the server it is connecting to + * is the same as the given digest provided in the connection string provided + * to it. + */ +public class ZKPeerX509TrustManager extends X509ExtendedTrustManager { private static final Logger LOG = LoggerFactory.getLogger(ZKPeerX509TrustManager.class); + private final ZKConfig zkConfig; private final InetSocketAddress peerAddr; // TODO: Host verification? private final String peerCertFingerPrintStr; public ZKPeerX509TrustManager( + final ZKConfig zkConfig, final InetSocketAddress peerAddr, final String peerCertFingerPrintStr) { + this.zkConfig = zkConfig; this.peerAddr = peerAddr; this.peerCertFingerPrintStr = peerCertFingerPrintStr; } public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[]{}; + return new X509Certificate[0]; + } + + + @Override + public void checkClientTrusted(final X509Certificate[] x509Certificates, + final String s) throws CertificateException { + LOG.error("Not supported!"); + throw new IllegalAccessError("Not Implemented!"); + } + + @Override + public void checkServerTrusted(final X509Certificate[] x509Certificates, + final String s) throws CertificateException { + if (x509Certificates.length == 0) { + final String errStr = "Invalid server, did not send any cert"; + LOG.error(errStr); + throw new CertificateException(errStr); + } + + validatePeerCert(x509Certificates[0]); + } + + @Override + public void checkClientTrusted(final X509Certificate[] x509Certificates, + final String s, final Socket socket) + throws CertificateException { + LOG.error("Not supported!"); + throw new IllegalAccessError("Not Implemented!"); + } + + @Override + public void checkServerTrusted(final X509Certificate[] x509Certificates, + final String s, final Socket socket) + throws CertificateException { + if (x509Certificates.length == 0) { + final String errStr = "Invalid server, did not send any cert"; + LOG.error(errStr); + throw new CertificateException(errStr); + } + + validatePeerCert(x509Certificates[0]); } - public void checkClientTrusted( - final X509Certificate[] certs, final String authType) + @Override + public void checkClientTrusted(final X509Certificate[] x509Certificates, + final String s, final SSLEngine sslEngine) throws CertificateException { LOG.error("Not supported!"); throw new IllegalAccessError("Not Implemented!"); } - public void checkServerTrusted( - final X509Certificate[] certs, final String authType) + @Override + public void checkServerTrusted(final X509Certificate[] x509Certificates, + final String s, final SSLEngine sslEngine) throws CertificateException { - if (certs.length == 0) { + if (x509Certificates.length == 0) { final String errStr = "Invalid server, did not send any cert"; LOG.error(errStr); throw new CertificateException(errStr); } - validatePeerCert(certs[0]); + validatePeerCert(x509Certificates[0]); } private void validatePeerCert(final X509Certificate cert) @@ -72,7 +125,7 @@ private void validatePeerCert(final X509Certificate cert) X509Util.verifySelfSigned(cert); try { - X509Util.validateCert(new ZKConfig(), peerCertFingerPrintStr, cert); + X509Util.validateCert(zkConfig, peerCertFingerPrintStr, cert); } catch (NoSuchAlgorithmException exp) { throw new CertificateException(exp); } diff --git a/src/java/main/org/apache/zookeeper/server/quorum/util/ZKX509TrustManager.java b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKX509TrustManager.java index 4a5fe023e65..24e73815216 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/util/ZKX509TrustManager.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKX509TrustManager.java @@ -17,46 +17,113 @@ */ package org.apache.zookeeper.server.quorum.util; -import java.security.InvalidKeyException; +import java.io.IOException; +import java.net.Socket; +import java.security.KeyStore; +import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; -import java.security.NoSuchProviderException; -import java.security.SignatureException; +import java.security.cert.Certificate; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Enumeration; import java.util.HashSet; import java.util.Set; -import javax.net.ssl.X509TrustManager; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509ExtendedTrustManager; +import org.apache.zookeeper.common.X509Exception; +import org.apache.zookeeper.common.X509Util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class ZKX509TrustManager implements X509TrustManager { +/** + * Given a TrustStore load all the certificate chains in it and verify + * the given certificate chain. This will be used by all modules if a + * TrustStore is indeed available. + * If a TrustStore is unavailable or no certificates are avaiable in + * TrustStore then verificate will fail. + */ +public class ZKX509TrustManager extends X509ExtendedTrustManager { private static final Logger LOG = LoggerFactory.getLogger(ZKX509TrustManager.class); - private final X509Certificate rootCACert; + private final Collection trustedCertList; - public ZKX509TrustManager(final X509Certificate rootCACert) { - this.rootCACert = rootCACert; + public ZKX509TrustManager(final String trustStoreLocation, + final String trustStorePassword) + throws X509Exception.TrustManagerException{ + this.trustedCertList = loadTrustAnchors(trustStoreLocation, + trustStorePassword); } + @Override public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[] { this.rootCACert }; + return new X509Certificate[0]; + } + + @Override + public void checkClientTrusted(final X509Certificate[] x509Certificates, + final String s) throws CertificateException { + try { + validateCertChain(x509Certificates); + } catch (CertificateVerificationException exp) { + throw new CertificateException(exp); + } + } + + @Override + public void checkServerTrusted(final X509Certificate[] x509Certificates, + final String s) throws + CertificateException { + try { + validateCertChain(x509Certificates); + } catch (CertificateVerificationException exp) { + throw new CertificateException(exp); + } + } + + @Override + public void checkClientTrusted(final X509Certificate[] x509Certificates, + final String s, final Socket socket) + throws CertificateException { + try { + validateCertChain(x509Certificates); + } catch (CertificateVerificationException exp) { + throw new CertificateException(exp); + } } - public void checkClientTrusted(X509Certificate[] certs, - String authType) throws CertificateException { + @Override + public void checkServerTrusted(final X509Certificate[] x509Certificates, + final String s, final Socket socket) + throws CertificateException { try { - validateCertChain(certs); + validateCertChain(x509Certificates); } catch (CertificateVerificationException exp) { throw new CertificateException(exp); } } - public void checkServerTrusted(X509Certificate[] certs, - String authType) throws CertificateException { + @Override + public void checkClientTrusted(final X509Certificate[] x509Certificates, + final String s, final SSLEngine sslEngine) + throws CertificateException { try { - validateCertChain(certs); + validateCertChain(x509Certificates); + } catch (CertificateVerificationException exp) { + throw new CertificateException(exp); + } + } + + @Override + public void checkServerTrusted(final X509Certificate[] x509Certificates, + final String s, final SSLEngine sslEngine) + throws CertificateException { + try { + validateCertChain(x509Certificates); } catch (CertificateVerificationException exp) { throw new CertificateException(exp); } @@ -64,6 +131,14 @@ public void checkServerTrusted(X509Certificate[] certs, private void validateCertChain(X509Certificate[] certs) throws CertificateVerificationException { + if (trustedCertList == null) { + throw new CertificateVerificationException("Failed to verify " + + "certificate due to lack of truststore."); + } + if (trustedCertList.size() == 0) { + throw new CertificateVerificationException("Failed to verify " + + "no trusted certificates provided."); + } X509Certificate clientCert = null; Set restOfCerts = new HashSet<>(); for (final X509Certificate cert: certs) { @@ -81,21 +156,43 @@ private void validateCertChain(X509Certificate[] certs) } } - restOfCerts.add(rootCACert); + restOfCerts.addAll(trustedCertList); CertificateVerifier.verifyCertificate(clientCert, restOfCerts); } - private void checkAllCerts(X509Certificate[] certs) - throws CertificateException { - for (X509Certificate cert : certs) { - try { - cert.verify(rootCACert.getPublicKey()); - } catch (NoSuchAlgorithmException - | InvalidKeyException | NoSuchProviderException - | SignatureException exp) { - LOG.error("cert validation failed, exp: " + exp); - throw new CertificateException(exp); + private Collection loadTrustAnchors( + final String storeLocation, final String storePassword) + throws X509Exception.TrustManagerException { + if (storeLocation == null) { + return null; + } + + if (storePassword == null) { + final String errStr = "Truststore password is required for the " + + "given truststore location: " + storeLocation; + LOG.error(errStr); + throw new X509Exception.TrustManagerException(errStr); + } + + final Collection trustedCertList = new ArrayList<>(); + try { + final KeyStore ts = X509Util.loadKeyStore(storeLocation, + storePassword); + final TrustManagerFactory tmf = + TrustManagerFactory.getInstance("SunX509"); + tmf.init(ts); + for (Enumeration e = ts.aliases(); e.hasMoreElements();) { + final String alias = e.nextElement(); + for (final Certificate cert : ts.getCertificateChain(alias)) { + trustedCertList.add((X509Certificate)cert); + } } + } catch (IOException | KeyStoreException | NoSuchAlgorithmException + | CertificateException e) { + final String errStr = "Could not load truststore: " + storeLocation; + LOG.error("{}", errStr, e); + throw new X509Exception.TrustManagerException(errStr, e); } + return trustedCertList; } } diff --git a/src/java/main/org/apache/zookeeper/server/util/ServerX509Util.java b/src/java/main/org/apache/zookeeper/server/util/ServerX509Util.java new file mode 100644 index 00000000000..31867ba9776 --- /dev/null +++ b/src/java/main/org/apache/zookeeper/server/util/ServerX509Util.java @@ -0,0 +1,64 @@ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.server.util; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.X509ExtendedTrustManager; + +import org.apache.zookeeper.common.X509ChainedTrustManager; +import org.apache.zookeeper.common.X509Exception; +import org.apache.zookeeper.common.X509Util; +import org.apache.zookeeper.common.ZKConfig; +import org.apache.zookeeper.server.ZookeeperServerConfig; +import org.apache.zookeeper.server.quorum.QuorumPeer; +import org.apache.zookeeper.server.quorum.util.ZKDynamicX509TrustManager; +import org.apache.zookeeper.server.quorum.util.ZKX509TrustManager; + +public class ServerX509Util extends X509Util { + /** + * SSL context which depend on dynamic config for authentication. Hence + * we need quorumPeer along with regular verification via Truststore done + * first. + * @param quorumPeer Used for getting QuorumVerifier and certs from + * QuorumPeerConfig. Both committed and last verified. + * @return SSLContext which can perform authentication based on dynamic cfg. + * @throws X509Exception.SSLContextException + */ + public static SSLContext createSSLContext(final QuorumPeer quorumPeer) + throws X509Exception.SSLContextException { + final ZKConfig zkConfig = new ZookeeperServerConfig(); + try { + return createSSLContext(zkConfig, + getTrustManager(zkConfig, quorumPeer)); + } catch (X509Exception.TrustManagerException exp) { + throw new X509Exception.SSLContextException(exp); + } + } + + public static X509ExtendedTrustManager getTrustManager( + final ZKConfig zkConfig, final QuorumPeer quorumPeer) + throws X509Exception.TrustManagerException { + return new X509ChainedTrustManager( + new ZKX509TrustManager(zkConfig.getProperty( + ZKConfig.SSL_TRUSTSTORE_LOCATION), + zkConfig.getProperty( + ZKConfig.SSL_TRUSTSTORE_PASSWD)), + new ZKDynamicX509TrustManager(quorumPeer)); + } +} From 351de6e5a97980bdba127ba5dbbf265ce8e979d1 Mon Sep 17 00:00:00 2001 From: Powell Molleti Date: Mon, 5 Sep 2016 23:12:35 -0700 Subject: [PATCH 07/12] Fail verification if Truststore has atleast one cert. With chained verification a truststore with atleast one cert trumps verification down stream. No more support for "cacert" in server string. --- .../main/org/apache/zookeeper/SSLCertCfg.java | 32 +----- .../common/FatalCertificateException.java | 35 ++++++ .../common/X509ChainedTrustManager.java | 54 ++++++++-- .../zookeeper/server/quorum/QuorumPeer.java | 102 ++++++++++++------ .../quorum/util/ZKX509TrustManager.java | 59 ++++------ 5 files changed, 183 insertions(+), 99 deletions(-) create mode 100644 src/java/main/org/apache/zookeeper/common/FatalCertificateException.java diff --git a/src/java/main/org/apache/zookeeper/SSLCertCfg.java b/src/java/main/org/apache/zookeeper/SSLCertCfg.java index 77027cf1244..af70d9f5a75 100644 --- a/src/java/main/org/apache/zookeeper/SSLCertCfg.java +++ b/src/java/main/org/apache/zookeeper/SSLCertCfg.java @@ -31,74 +31,52 @@ public class SSLCertCfg { private static final Logger LOG = LoggerFactory.getLogger(SSLCertCfg.class); - private final CertType certType; private final String certFingerPrintStr; - public enum CertType { - NONE, SELF, CA; - } public SSLCertCfg() { - certType = CertType.NONE; certFingerPrintStr = null; } - public SSLCertCfg(final CertType certType, - final String certFingerPrintStr) { - this.certType = certType; + public SSLCertCfg(final String certFingerPrintStr) { this.certFingerPrintStr = certFingerPrintStr; } - public boolean isSelfSigned() { - return certType == CertType.SELF; - } - - public boolean isCASigned() { - return certType == CertType.CA; - } - public String getCertFingerPrintStr() { return certFingerPrintStr; } public static SSLCertCfg parseCertCfgStr(final String certCfgStr) throws QuorumPeerConfig.ConfigException { - SSLCertCfg.CertType certType = SSLCertCfg.CertType.NONE; int fpIndex = Integer.MAX_VALUE; final String[] parts = certCfgStr.split(":"); final Map propKvMap = getKeyAndIndexMap(certCfgStr); - if (propKvMap.containsKey("cert") && - propKvMap.containsKey("cacert")) { + if (propKvMap.containsKey("cert")) { final String errStr = "Server string has both self signed " + "cert and ca cert: " + certCfgStr; throw new QuorumPeerConfig.ConfigException(errStr); } else if (propKvMap.containsKey("cert")) { - certType = SSLCertCfg.CertType.SELF; fpIndex = propKvMap.get("cert") + 1; if (parts.length < fpIndex) { final String errStr = "No fingerprint provided for self " + "signed, server cfg string: " + certCfgStr; throw new QuorumPeerConfig.ConfigException(errStr); } - } else if (propKvMap.containsKey("cacert")) { - certType = SSLCertCfg.CertType.SELF; - fpIndex = propKvMap.get("cacert") + 1; } if (fpIndex != Integer.MAX_VALUE && parts.length > fpIndex) { - LOG.debug("certCfgStr: " + certCfgStr + ", cert type:" + certType + + LOG.debug("certCfgStr: " + certCfgStr + ", fp:" + parts[fpIndex]); if (getMessageDigest(parts[fpIndex]) != null) { - return new SSLCertCfg(certType, parts[fpIndex]); + return new SSLCertCfg(parts[fpIndex]); } } return new SSLCertCfg(); } - private static Map getKeyAndIndexMap( - final String cfgStr) { + private static Map getKeyAndIndexMap(final String cfgStr) { final Map propKvMap = new HashMap<>(); final String[] parts = cfgStr.split(":"); for (int i = 0; i < parts.length; i++) { diff --git a/src/java/main/org/apache/zookeeper/common/FatalCertificateException.java b/src/java/main/org/apache/zookeeper/common/FatalCertificateException.java new file mode 100644 index 00000000000..c1868658003 --- /dev/null +++ b/src/java/main/org/apache/zookeeper/common/FatalCertificateException.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.common; + +import java.security.cert.CertificateException; + +@SuppressWarnings("serial") +public class FatalCertificateException extends CertificateException { + public FatalCertificateException(String message) { + super(message); + } + + public FatalCertificateException(Throwable cause) { + super(cause); + } + + public FatalCertificateException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/java/main/org/apache/zookeeper/common/X509ChainedTrustManager.java b/src/java/main/org/apache/zookeeper/common/X509ChainedTrustManager.java index 00ba25e28ff..0090cd20804 100644 --- a/src/java/main/org/apache/zookeeper/common/X509ChainedTrustManager.java +++ b/src/java/main/org/apache/zookeeper/common/X509ChainedTrustManager.java @@ -81,7 +81,14 @@ public void checkClientTrusted(final X509Certificate[] x509Certificates, .checkClientTrusted(x509Certificates, s); return; // means success, i.e no exception raised. } catch(CertificateException exp) { - // Ignore each exception. + // Ignore each exception except the one that is fatal! + if (exp instanceof FatalCertificateException) { + final String errStr = + "Could not verify certificate chain: " + + String.join(" ", expErrMsgList); + LOG.error("{}", errStr, exp); + throw new CertificateException(errStr, exp); + } expErrMsgList.add(exp.getMessage()); } } @@ -104,7 +111,14 @@ public void checkServerTrusted(final X509Certificate[] x509Certificates, .checkServerTrusted(x509Certificates, s); return; // means success, i.e no exception raised. } catch(CertificateException exp) { - // Ignore each exception. + // Ignore each exception except the one that is fatal! + if (exp instanceof FatalCertificateException) { + final String errStr = + "Could not verify certificate chain: " + + String.join(" ", expErrMsgList); + LOG.error("{}", errStr, exp); + throw new CertificateException(errStr, exp); + } expErrMsgList.add(exp.getMessage()); } } @@ -133,7 +147,14 @@ public void checkClientTrusted(final X509Certificate[] x509Certificates, .checkClientTrusted(x509Certificates, s, socket); return; // means success, i.e no exception raised. } catch(CertificateException exp) { - // Ignore each exception. + // Ignore each exception except the one that is fatal! + if (exp instanceof FatalCertificateException) { + final String errStr = + "Could not verify certificate chain: " + + String.join(" ", expErrMsgList); + LOG.error("{}", errStr, exp); + throw new CertificateException(errStr, exp); + } expErrMsgList.add(exp.getMessage()); } } @@ -157,7 +178,14 @@ public void checkServerTrusted(final X509Certificate[] x509Certificates, .checkServerTrusted(x509Certificates, s, socket); return; // means success, i.e no exception raised. } catch(CertificateException exp) { - // Ignore each exception. + // Ignore each exception except the one that is fatal! + if (exp instanceof FatalCertificateException) { + final String errStr = + "Could not verify certificate chain: " + + String.join(" ", expErrMsgList); + LOG.error("{}", errStr, exp); + throw new CertificateException(errStr, exp); + } expErrMsgList.add(exp.getMessage()); } } @@ -181,7 +209,14 @@ public void checkClientTrusted(final X509Certificate[] x509Certificates, .checkClientTrusted(x509Certificates, s, sslEngine); return; // means success, i.e no exception raised. } catch(CertificateException exp) { - // Ignore each exception. + // Ignore each exception except the one that is fatal! + if (exp instanceof FatalCertificateException) { + final String errStr = + "Could not verify certificate chain: " + + String.join(" ", expErrMsgList); + LOG.error("{}", errStr, exp); + throw new CertificateException(errStr, exp); + } expErrMsgList.add(exp.getMessage()); } } @@ -205,7 +240,14 @@ public void checkServerTrusted(final X509Certificate[] x509Certificates, .checkServerTrusted(x509Certificates, s, sslEngine); return; // means success, i.e no exception raised. } catch(CertificateException exp) { - // Ignore each exception. + // Ignore each exception except the one that is fatal! + if (exp instanceof FatalCertificateException) { + final String errStr = + "Could not verify certificate chain: " + + String.join(" ", expErrMsgList); + LOG.error("{}", errStr, exp); + throw new CertificateException(errStr, exp); + } expErrMsgList.add(exp.getMessage()); } } diff --git a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java index 2e0e0d91e26..820e721a6d0 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java @@ -37,6 +37,7 @@ import java.security.cert.CertificateEncodingException; import java.security.cert.X509Certificate; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -125,6 +126,7 @@ public static class QuorumServer { public InetSocketAddress electionAddr = null; public InetSocketAddress clientAddr = null; + public InetSocketAddress secureClientAddr = null; public long id; @@ -239,22 +241,21 @@ public QuorumServer(long sid, String addressStr) throws ConfigException { throw new ConfigException(addressStr + wrongFormat); } - if (serverClientParts.length == 2) { - //LOG.warn("ClientParts: " + serverClientParts[1]); - String clientParts[] = splitWithLeadingHostname(serverClientParts[1]); - if (clientParts.length > 2) { - throw new ConfigException(addressStr + wrongFormat); + // Compatible part is host:port or just port which is used for non + // secure socket + // Support for both sockets or either of them is as follows + // plain:host:port;secure:host:port and similar config for ipv6 + if (serverClientParts.length > 1) { + parseHostString(serverClientParts[1], addressStr); + if (serverClientParts.length > 2) { + parseHostString(serverClientParts[2], addressStr); } + } - // is client_config a host:port or just a port - String hostname = (clientParts.length == 2) ? clientParts[0] : "0.0.0.0"; - try { - clientAddr = new InetSocketAddress(hostname, - Integer.parseInt(clientParts[clientParts.length - 1])); - //LOG.warn("Set clientAddr to " + clientAddr); - } catch (NumberFormatException e) { - throw new ConfigException("Address unresolved: " + hostname + ":" + clientParts[clientParts.length - 1]); - } + if (serverClientParts.length > 3) { + throw new ConfigException("Invalid server string, more than " + + "two sections parsed after ; for given string: " + + addressStr); } // server_config should be either host:port:port or host:port:port:type @@ -272,14 +273,13 @@ public QuorumServer(long sid, String addressStr) throws ConfigException { } // Now we can expect the following - // participant:cert::cacert: + // participant:cert: this.sslCertCfg = new SSLCertCfg(); if (serverParts.length == 4) { // backward compatibility first. setType(serverParts[3]); } else if (serverParts.length >= 4) { - // Parse this: cert:SHA-256-XXXX or cacert:SHA-256-XXX - // cert is self signed, cacert's fingerprint is optional. + // Parse this: cert:SHA-256-XXXX final HashMap propKvMap = new HashMap<>(); for (int i = 3; i < serverParts.length; i++) { propKvMap.put(serverParts[i].trim().toLowerCase(), i); @@ -317,6 +317,50 @@ private void setMyAddrs() { this.myAddrs = excludedSpecialAddresses(this.myAddrs); } + private static InetSocketAddress parseCompatibleHostString( + final String hostStr, final String serverStr) + throws ConfigException { + //LOG.warn("ClientParts: " + serverClientParts[1]); + final String clientParts[] = splitWithLeadingHostname(hostStr); + if (clientParts.length > 2) { + throw new ConfigException(serverStr + wrongFormat); + } + + // is client_config a host:port or just a port + final String hostname = (clientParts.length == 2) ? + clientParts[0] : "0.0.0.0"; + try { + return new InetSocketAddress(hostname, + Integer.parseInt(clientParts[clientParts.length - 1])); + //LOG.warn("Set clientAddr to " + clientAddr); + } catch (NumberFormatException e) { + throw new ConfigException("Address unresolved: " + hostname + + ":" + clientParts[clientParts.length - 1]); + } + } + + private void parseHostString(final String hostStr, + final String serverStr) + throws ConfigException { + final String[] parts = hostStr.split(":"); + if (parts[0].toLowerCase().equals("plain")) { + clientAddr = parseCompatibleHostString( + removeLeadingItem(parts), serverStr); + } else if (parts[0].toLowerCase().equals("secure")) { + secureClientAddr = parseCompatibleHostString( + removeLeadingItem(parts), serverStr); + } else { + clientAddr = parseCompatibleHostString(hostStr, serverStr); + } + } + + private String removeLeadingItem(final String[] parts) { + final List partsList = + new ArrayList<>(Arrays.asList(parts)); + partsList.remove(0); + return String.join(":", partsList); + } + private static String delimitedHostString(InetSocketAddress addr) { String host = addr.getHostString(); @@ -341,30 +385,28 @@ public String toString(){ } if (type == LearnerType.OBSERVER) sw.append(":observer"); else if (type == LearnerType.PARTICIPANT) sw.append(":participant"); - if (isSelfSigned()) { + if (hasCertFp()) { sw.append(":cert:"); sw.append(sslCertCfg.getCertFingerPrintStr()); - } else if (isCASigned()) { - sw.append(":cacert:"); - if (sslCertCfg.getCertFingerPrintStr() != null) { - sw.append(sslCertCfg.getCertFingerPrintStr()); - } } if (clientAddr!=null){ - sw.append(";"); + sw.append(";plain:"); sw.append(delimitedHostString(clientAddr)); sw.append(":"); sw.append(String.valueOf(clientAddr.getPort())); } + if (secureClientAddr != null) { + sw.append(";secure:"); + sw.append(delimitedHostString(secureClientAddr)); + sw.append(":"); + sw.append(String.valueOf(secureClientAddr.getPort())); + } return sw.toString(); } - public boolean isSelfSigned() { - return sslCertCfg != null && sslCertCfg.isSelfSigned(); - } - - public boolean isCASigned() { - return sslCertCfg != null && sslCertCfg.isCASigned(); + public boolean hasCertFp() { + return sslCertCfg != null && sslCertCfg.getCertFingerPrintStr() + != null; } public int hashCode() { diff --git a/src/java/main/org/apache/zookeeper/server/quorum/util/ZKX509TrustManager.java b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKX509TrustManager.java index 24e73815216..d4871d894f2 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/util/ZKX509TrustManager.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKX509TrustManager.java @@ -35,6 +35,7 @@ import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509ExtendedTrustManager; +import org.apache.zookeeper.common.FatalCertificateException; import org.apache.zookeeper.common.X509Exception; import org.apache.zookeeper.common.X509Util; import org.slf4j.Logger; @@ -44,8 +45,11 @@ * Given a TrustStore load all the certificate chains in it and verify * the given certificate chain. This will be used by all modules if a * TrustStore is indeed available. - * If a TrustStore is unavailable or no certificates are avaiable in - * TrustStore then verificate will fail. + * If a TrustStore is unavailable or no certificates are available in + * TrustStore then verification will fail but this failure can be considered as + * not fatal. + * If a TrustStore has at-least one cert and verification fails then it is + * considered a fatal failure. */ public class ZKX509TrustManager extends X509ExtendedTrustManager { private static final Logger LOG @@ -67,76 +71,52 @@ public X509Certificate[] getAcceptedIssuers() { @Override public void checkClientTrusted(final X509Certificate[] x509Certificates, final String s) throws CertificateException { - try { - validateCertChain(x509Certificates); - } catch (CertificateVerificationException exp) { - throw new CertificateException(exp); - } + validateCertChain(x509Certificates); } @Override public void checkServerTrusted(final X509Certificate[] x509Certificates, final String s) throws CertificateException { - try { - validateCertChain(x509Certificates); - } catch (CertificateVerificationException exp) { - throw new CertificateException(exp); - } + validateCertChain(x509Certificates); } @Override public void checkClientTrusted(final X509Certificate[] x509Certificates, final String s, final Socket socket) throws CertificateException { - try { - validateCertChain(x509Certificates); - } catch (CertificateVerificationException exp) { - throw new CertificateException(exp); - } + validateCertChain(x509Certificates); } @Override public void checkServerTrusted(final X509Certificate[] x509Certificates, final String s, final Socket socket) throws CertificateException { - try { - validateCertChain(x509Certificates); - } catch (CertificateVerificationException exp) { - throw new CertificateException(exp); - } + validateCertChain(x509Certificates); } @Override public void checkClientTrusted(final X509Certificate[] x509Certificates, final String s, final SSLEngine sslEngine) throws CertificateException { - try { - validateCertChain(x509Certificates); - } catch (CertificateVerificationException exp) { - throw new CertificateException(exp); - } + validateCertChain(x509Certificates); } @Override public void checkServerTrusted(final X509Certificate[] x509Certificates, final String s, final SSLEngine sslEngine) throws CertificateException { - try { - validateCertChain(x509Certificates); - } catch (CertificateVerificationException exp) { - throw new CertificateException(exp); - } + validateCertChain(x509Certificates); } private void validateCertChain(X509Certificate[] certs) - throws CertificateVerificationException { + throws CertificateException { if (trustedCertList == null) { - throw new CertificateVerificationException("Failed to verify " + + throw new CertificateException("Failed to verify " + "certificate due to lack of truststore."); } if (trustedCertList.size() == 0) { - throw new CertificateVerificationException("Failed to verify " + + throw new CertificateException("Failed to verify " + "no trusted certificates provided."); } X509Certificate clientCert = null; @@ -157,7 +137,14 @@ private void validateCertChain(X509Certificate[] certs) } restOfCerts.addAll(trustedCertList); - CertificateVerifier.verifyCertificate(clientCert, restOfCerts); + try { + CertificateVerifier.verifyCertificate(clientCert, restOfCerts); + } catch (CertificateVerificationException exp) { + final String errStr = "verification with few trust anchors failed" + + ". Error: " + exp.getMessage(); + LOG.error("{}", errStr, exp); + throw new FatalCertificateException(errStr, exp); + } } private Collection loadTrustAnchors( From c8f16214609948834a9cc087294cabc217124f8b Mon Sep 17 00:00:00 2001 From: Powell Molleti Date: Tue, 6 Sep 2016 01:20:22 -0700 Subject: [PATCH 08/12] Self signed certs are not verified via Truststore. TODO: Fix this later. --- .../main/org/apache/zookeeper/SSLCertCfg.java | 4 ---- .../org/apache/zookeeper/common/X509Util.java | 14 ++++------- .../org/apache/zookeeper/common/ZKConfig.java | 23 ++++++++++++------- .../server/ZookeeperServerConfig.java | 12 +++------- .../zookeeper/server/quorum/QuorumPeer.java | 5 +++- .../server/quorum/QuorumPeerConfig.java | 13 +++-------- .../quorum/util/ZKX509TrustManager.java | 21 +++++++++-------- 7 files changed, 40 insertions(+), 52 deletions(-) diff --git a/src/java/main/org/apache/zookeeper/SSLCertCfg.java b/src/java/main/org/apache/zookeeper/SSLCertCfg.java index af70d9f5a75..5bdd4e27b9d 100644 --- a/src/java/main/org/apache/zookeeper/SSLCertCfg.java +++ b/src/java/main/org/apache/zookeeper/SSLCertCfg.java @@ -52,10 +52,6 @@ public static SSLCertCfg parseCertCfgStr(final String certCfgStr) final Map propKvMap = getKeyAndIndexMap(certCfgStr); if (propKvMap.containsKey("cert")) { - final String errStr = "Server string has both self signed " + - "cert and ca cert: " + certCfgStr; - throw new QuorumPeerConfig.ConfigException(errStr); - } else if (propKvMap.containsKey("cert")) { fpIndex = propKvMap.get("cert") + 1; if (parts.length < fpIndex) { final String errStr = "No fingerprint provided for self " + diff --git a/src/java/main/org/apache/zookeeper/common/X509Util.java b/src/java/main/org/apache/zookeeper/common/X509Util.java index 6d705db19a0..8a97ba03931 100644 --- a/src/java/main/org/apache/zookeeper/common/X509Util.java +++ b/src/java/main/org/apache/zookeeper/common/X509Util.java @@ -20,7 +20,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; -import java.net.InetSocketAddress; import java.security.InvalidKeyException; import java.security.KeyStore; import java.security.KeyStoreException; @@ -38,24 +37,17 @@ import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; -import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509ExtendedTrustManager; import javax.net.ssl.X509KeyManager; -import javax.net.ssl.X509TrustManager; import javax.xml.bind.DatatypeConverter; import org.apache.zookeeper.SSLCertCfg; -import org.apache.zookeeper.server.quorum.QuorumPeer; -import org.apache.zookeeper.server.quorum.util.ZKDynamicX509TrustManager; -import org.apache.zookeeper.server.quorum.util.ZKPeerX509TrustManager; -import org.apache.zookeeper.server.quorum.util.ZKX509TrustManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static javax.xml.bind.DatatypeConverter.printHexBinary; import static org.apache.zookeeper.common.X509Exception.KeyManagerException; import static org.apache.zookeeper.common.X509Exception.SSLContextException; -import static org.apache.zookeeper.common.X509Exception.TrustManagerException; /** * Utility code for X509 handling @@ -73,8 +65,10 @@ public static SSLContext createSSLContext( } protected static KeyManager[] createKeyManagers(final ZKConfig config) - throws - SSLContextException { + throws SSLContextException { + LOG.info("keystore key: " + ZKConfig.SSL_KEYSTORE_LOCATION); + LOG.info("keystore pwd: " + ZKConfig.SSL_KEYSTORE_PASSWD); + final String keyStoreLocationProp = config.getProperty(ZKConfig.SSL_KEYSTORE_LOCATION); final String keyStorePasswordProp = diff --git a/src/java/main/org/apache/zookeeper/common/ZKConfig.java b/src/java/main/org/apache/zookeeper/common/ZKConfig.java index 3ab5009334c..32fa730533b 100644 --- a/src/java/main/org/apache/zookeeper/common/ZKConfig.java +++ b/src/java/main/org/apache/zookeeper/common/ZKConfig.java @@ -45,7 +45,7 @@ public abstract class ZKConfig { public static final String SSL_VERSION_DEFAULT = "TLSv1"; public static final String SSL_VERSION = "ssl.version"; - public static final String SSL_KEYSTORE_LOCATION = "ssl.keyStore"; + public static final String SSL_KEYSTORE_LOCATION = "ssl.keyStore.location"; public static final String SSL_KEYSTORE_PASSWD = "ssl.keyStore.password"; public static final String SSL_TRUSTSTORE_LOCATION = "ssl.trustStore.location"; @@ -56,12 +56,14 @@ public abstract class ZKConfig { public static final String SSL_DIGEST_ALGOS = "ssl.digest.algos"; protected final Map properties = new HashMap(); + private final SslConfig sslConfig; /** * properties, which are common to both client and server, are initialized * from system properties */ - public ZKConfig() { + public ZKConfig(final SslConfig sslConfig) { + this.sslConfig = sslConfig; init(); } @@ -72,8 +74,9 @@ public ZKConfig() { * if failed to load configuration properties */ - public ZKConfig(String configPath) throws ConfigException { - this(new File(configPath)); + public ZKConfig(final String configPath, final SslConfig sslConfig) + throws ConfigException { + this(new File(configPath), sslConfig); } /** @@ -83,8 +86,9 @@ public ZKConfig(String configPath) throws ConfigException { * @throws ConfigException * if failed to load configuration properties */ - public ZKConfig(File configFile) throws ConfigException { - this(); + public ZKConfig(final File configFile, final SslConfig sslConfig) + throws ConfigException { + this(sslConfig); addConfiguration(configFile); } @@ -105,6 +109,7 @@ protected void handleBackwardCompatibility() { properties.put(SSL_VERSION, systemPropDefault( getSslConfig().getSslVersion(), getSslConfig().getSslVersionDefault())); + LOG.info("registering: " + getSslConfig().getSslKeyStoreLocation()); properties.put(SSL_KEYSTORE_LOCATION, System.getProperty(getSslConfig().getSslKeyStoreLocation())); properties.put(SSL_KEYSTORE_PASSWD, @@ -114,13 +119,15 @@ protected void handleBackwardCompatibility() { properties.put(SSL_TRUSTSTORE_PASSWD, System.getProperty(getSslConfig().getSslTrustStorePassword())); properties.put(SSL_AUTHPROVIDER, - getSslConfig().getSslAuthProvider()); + System.getProperty(getSslConfig().getSslAuthProvider())); properties.put(SSL_DIGEST_ALGOS, systemPropDefault( getSslConfig().getSslDigestAlgos(), getSslConfig().getSslDefaultDigestAlgo())); } - protected abstract SslConfig getSslConfig(); + private SslConfig getSslConfig() { + return sslConfig; + } protected static String systemPropDefault(final String key, final String defaultValue) { diff --git a/src/java/main/org/apache/zookeeper/server/ZookeeperServerConfig.java b/src/java/main/org/apache/zookeeper/server/ZookeeperServerConfig.java index 935e83190c2..3ac51973b68 100644 --- a/src/java/main/org/apache/zookeeper/server/ZookeeperServerConfig.java +++ b/src/java/main/org/apache/zookeeper/server/ZookeeperServerConfig.java @@ -19,7 +19,6 @@ import java.io.File; -import org.apache.zookeeper.common.SslConfig; import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.server.quorum.QuorumPeerConfig; @@ -35,17 +34,17 @@ public class ZookeeperServerConfig extends ZKConfig { new ZookeeperServerSslConfig(); public ZookeeperServerConfig() { - super(); + super(new ZookeeperServerSslConfig()); } public ZookeeperServerConfig(File configFile) throws QuorumPeerConfig.ConfigException { - super(configFile); + super(configFile, new ZookeeperServerSslConfig()); } public ZookeeperServerConfig(String configPath) throws QuorumPeerConfig.ConfigException { - super(configPath); + super(configPath, new ZookeeperServerSslConfig()); } /** @@ -61,9 +60,4 @@ protected void handleBackwardCompatibility() { properties.put(KINIT_COMMAND, System.getProperty(KINIT_COMMAND)); properties.put(JGSS_NATIVE, System.getProperty(JGSS_NATIVE)); } - - @Override - protected SslConfig getSslConfig() { - return sslConfig; - } } diff --git a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java index 820e721a6d0..8ce84bb1a0e 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java @@ -390,7 +390,10 @@ public String toString(){ sw.append(sslCertCfg.getCertFingerPrintStr()); } if (clientAddr!=null){ - sw.append(";plain:"); + sw.append(";"); + if (secureClientAddr != null) { + sw.append("plain:"); + } sw.append(delimitedHostString(clientAddr)); sw.append(":"); sw.append(String.valueOf(clientAddr.getPort())); diff --git a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java index e64348d1071..9082569cd66 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java @@ -40,7 +40,6 @@ import org.apache.zookeeper.common.AtomicFileWritingIdiom.OutputStreamStatement; import org.apache.zookeeper.common.AtomicFileWritingIdiom.WriterStatement; import org.apache.zookeeper.common.PathUtils; -import org.apache.zookeeper.common.SslConfig; import org.apache.zookeeper.common.StringUtils; import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.server.ZooKeeperServer; @@ -61,7 +60,6 @@ public class QuorumPeerConfig extends ZKConfig { private static final int UNSET_SERVERID = -1; public static final String nextDynamicConfigFileSuffix = ".dynamic.next"; - private final QuorumSslConfig sslConfig = new QuorumSslConfig(); private static boolean standaloneEnabled = true; private static boolean reconfigEnabled = false; @@ -112,22 +110,17 @@ public ConfigException(String msg, Exception e) { } public QuorumPeerConfig() { - super(); + super(new QuorumSslConfig()); } public QuorumPeerConfig(File configFile) throws QuorumPeerConfig.ConfigException { - super(configFile); + super(configFile, new QuorumSslConfig()); } public QuorumPeerConfig(String configPath) throws QuorumPeerConfig.ConfigException { - super(configPath); - } - - @Override - protected SslConfig getSslConfig() { - return sslConfig; + super(configPath, new QuorumSslConfig()); } /** diff --git a/src/java/main/org/apache/zookeeper/server/quorum/util/ZKX509TrustManager.java b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKX509TrustManager.java index d4871d894f2..3773ba40228 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/util/ZKX509TrustManager.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/util/ZKX509TrustManager.java @@ -119,21 +119,22 @@ private void validateCertChain(X509Certificate[] certs) throw new CertificateException("Failed to verify " + "no trusted certificates provided."); } - X509Certificate clientCert = null; - Set restOfCerts = new HashSet<>(); + + final X509Certificate clientCert = certs[0]; + if (CertificateVerifier.isSelfSigned(clientCert)) { + throw new CertificateException("Not going to verify self-signed " + + "certs for now using truststore."); + } + + final Set restOfCerts = new HashSet<>(); for (final X509Certificate cert: certs) { // skip self signed cert - if (CertificateVerifier.isSelfSigned(cert)) { + if (CertificateVerifier.isSelfSigned(cert) || + clientCert.equals(cert)) { continue; } - // first non self signed cert will be client cert? - // TODO: how do we get the first cert?. - if (clientCert == null) { - clientCert = cert; - } else { - restOfCerts.add(cert); - } + restOfCerts.add(cert); } restOfCerts.addAll(trustedCertList); From f736859e06d045bd04aaa9227021d98d938991c5 Mon Sep 17 00:00:00 2001 From: Powell Molleti Date: Wed, 14 Sep 2016 00:48:30 -0700 Subject: [PATCH 09/12] Server config string has both plain and secure address Now the client parts of server string could look like this: plain:127.0.1.1:2181;secure:127.0.1.1:2281 Its upto caller to ensure secureClientAddress and secureClientPortAddress are same in the secure section above. --- .../main/org/apache/zookeeper/server/quorum/QuorumPeer.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java index 8ce84bb1a0e..992a22bbb8f 100644 --- a/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java +++ b/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java @@ -229,14 +229,15 @@ private static String[] splitWithLeadingHostname(String s) } private static final String wrongFormat = " does not have the form server_config or server_config;client_config"+ - " where server_config is host:port:port or host:port:port:type and client_config is port or host:port"; + " where server_config is host:port:port or host:port:port:type and " + + "client_config is port or host:port or plain:host:port;secure:host:port"; public QuorumServer(long sid, String addressStr) throws ConfigException { // LOG.warn("sid = " + sid + " addressStr = " + addressStr); this.id = sid; String serverClientParts[] = addressStr.split(";"); String serverParts[] = splitWithLeadingHostname(serverClientParts[0]); - if ((serverClientParts.length > 2) || (serverParts.length < 3) + if ((serverClientParts.length > 3) || (serverParts.length < 3) || (serverParts.length > 6)) { throw new ConfigException(addressStr + wrongFormat); } From 0b0ce20081832c1630d6da9679cbde81b0ecf413 Mon Sep 17 00:00:00 2001 From: Powell Molleti Date: Wed, 14 Sep 2016 22:17:13 -0700 Subject: [PATCH 10/12] Fix cert parsing a bit. --- src/java/main/org/apache/zookeeper/SSLCertCfg.java | 9 ++------- .../main/org/apache/zookeeper/client/ClientX509Util.java | 6 ++++++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/java/main/org/apache/zookeeper/SSLCertCfg.java b/src/java/main/org/apache/zookeeper/SSLCertCfg.java index 5bdd4e27b9d..8783d0669fa 100644 --- a/src/java/main/org/apache/zookeeper/SSLCertCfg.java +++ b/src/java/main/org/apache/zookeeper/SSLCertCfg.java @@ -47,21 +47,16 @@ public String getCertFingerPrintStr() { public static SSLCertCfg parseCertCfgStr(final String certCfgStr) throws QuorumPeerConfig.ConfigException { - int fpIndex = Integer.MAX_VALUE; final String[] parts = certCfgStr.split(":"); final Map propKvMap = getKeyAndIndexMap(certCfgStr); if (propKvMap.containsKey("cert")) { - fpIndex = propKvMap.get("cert") + 1; - if (parts.length < fpIndex) { + int fpIndex = propKvMap.get("cert") + 1; + if (parts.length <= fpIndex) { final String errStr = "No fingerprint provided for self " + "signed, server cfg string: " + certCfgStr; throw new QuorumPeerConfig.ConfigException(errStr); } - } - - if (fpIndex != Integer.MAX_VALUE && - parts.length > fpIndex) { LOG.debug("certCfgStr: " + certCfgStr + ", fp:" + parts[fpIndex]); if (getMessageDigest(parts[fpIndex]) != null) { diff --git a/src/java/main/org/apache/zookeeper/client/ClientX509Util.java b/src/java/main/org/apache/zookeeper/client/ClientX509Util.java index bc2edc72c6e..c5004663dda 100644 --- a/src/java/main/org/apache/zookeeper/client/ClientX509Util.java +++ b/src/java/main/org/apache/zookeeper/client/ClientX509Util.java @@ -42,6 +42,12 @@ public static SSLContext createSSLContext( final ZKConfig zkConfig, final InetSocketAddress peerAddr, final String peerCertFingerPrintStr) throws X509Exception.SSLContextException { + if (peerCertFingerPrintStr == null) { + final String errStr = "Peer's: " + peerAddr + + " certificate fingerprint cannot be null"; + LOG.error(errStr); + throw new IllegalAccessError(errStr); + } try { return createSSLContext(zkConfig, new X509ChainedTrustManager( new ZKX509TrustManager(zkConfig.getProperty( From 0ccee0ac33261bc55c49f06cf51cb9a24ea06c63 Mon Sep 17 00:00:00 2001 From: Powell Molleti Date: Mon, 6 Mar 2017 20:19:41 -0800 Subject: [PATCH 11/12] Fix test code due to Quorum Peer constructor change and also since JUTE_MAXBUFFER is moved. --- .../zookeeper/ClientCnxnSocketTest.java | 10 ++--- .../zookeeper/client/ZKClientConfigTest.java | 38 +++++++++---------- .../quorum/ReconfigDuringLeaderSyncTest.java | 12 +++--- 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/java/test/org/apache/zookeeper/ClientCnxnSocketTest.java b/src/java/test/org/apache/zookeeper/ClientCnxnSocketTest.java index 054e1ed4803..45868f8c514 100644 --- a/src/java/test/org/apache/zookeeper/ClientCnxnSocketTest.java +++ b/src/java/test/org/apache/zookeeper/ClientCnxnSocketTest.java @@ -17,22 +17,22 @@ */ package org.apache.zookeeper; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import java.io.IOException; import org.apache.zookeeper.client.ZKClientConfig; -import org.apache.zookeeper.common.ZKConfig; import org.junit.Test; +import static org.apache.zookeeper.server.ZookeeperServerConfig.JUTE_MAXBUFFER; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + public class ClientCnxnSocketTest { @Test public void testWhenInvalidJuteMaxBufferIsConfiguredIOExceptionIsThrown() { ZKClientConfig clientConfig = new ZKClientConfig(); String value = "SomeInvalidInt"; - clientConfig.setProperty(ZKConfig.JUTE_MAXBUFFER, value); + clientConfig.setProperty(JUTE_MAXBUFFER, value); // verify ClientCnxnSocketNIO creation try { new ClientCnxnSocketNIO(clientConfig); diff --git a/src/java/test/org/apache/zookeeper/client/ZKClientConfigTest.java b/src/java/test/org/apache/zookeeper/client/ZKClientConfigTest.java index 98f7c51bc96..2f61edab683 100644 --- a/src/java/test/org/apache/zookeeper/client/ZKClientConfigTest.java +++ b/src/java/test/org/apache/zookeeper/client/ZKClientConfigTest.java @@ -18,18 +18,6 @@ package org.apache.zookeeper.client; -import static org.apache.zookeeper.client.ZKClientConfig.DISABLE_AUTO_WATCH_RESET; -import static org.apache.zookeeper.client.ZKClientConfig.ENABLE_CLIENT_SASL_KEY; -import static org.apache.zookeeper.client.ZKClientConfig.LOGIN_CONTEXT_NAME_KEY; -import static org.apache.zookeeper.client.ZKClientConfig.SECURE_CLIENT; -import static org.apache.zookeeper.client.ZKClientConfig.ZK_SASL_CLIENT_USERNAME; -import static org.apache.zookeeper.client.ZKClientConfig.ZOOKEEPER_CLIENT_CNXN_SOCKET; -import static org.apache.zookeeper.client.ZKClientConfig.ZOOKEEPER_SERVER_REALM; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -39,13 +27,25 @@ import java.util.Properties; import java.util.concurrent.TimeUnit; -import org.apache.zookeeper.common.ZKConfig; import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; +import static org.apache.zookeeper.client.ZKClientConfig.DISABLE_AUTO_WATCH_RESET; +import static org.apache.zookeeper.client.ZKClientConfig.ENABLE_CLIENT_SASL_KEY; +import static org.apache.zookeeper.client.ZKClientConfig.LOGIN_CONTEXT_NAME_KEY; +import static org.apache.zookeeper.client.ZKClientConfig.SECURE_CLIENT; +import static org.apache.zookeeper.client.ZKClientConfig.ZK_SASL_CLIENT_USERNAME; +import static org.apache.zookeeper.client.ZKClientConfig.ZOOKEEPER_CLIENT_CNXN_SOCKET; +import static org.apache.zookeeper.client.ZKClientConfig.ZOOKEEPER_SERVER_REALM; +import static org.apache.zookeeper.server.ZookeeperServerConfig.JUTE_MAXBUFFER; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + public class ZKClientConfigTest { private static final File testData = new File(System.getProperty("test.data.dir", "build/test/data")); @Rule @@ -165,9 +165,9 @@ public void testIntegerRetrievalFromProperty() { // property is set but can not be parsed to int, we should get the // NumberFormatException - conf.setProperty(ZKConfig.JUTE_MAXBUFFER, "InvlaidIntValue123"); + conf.setProperty(JUTE_MAXBUFFER, "InvlaidIntValue123"); try { - result = conf.getInt(ZKConfig.JUTE_MAXBUFFER, defaultValue); + result = conf.getInt(JUTE_MAXBUFFER, defaultValue); fail("NumberFormatException is expected"); } catch (NumberFormatException exception) { // do nothing @@ -176,15 +176,15 @@ public void testIntegerRetrievalFromProperty() { // property is set to an valid int, we should get the set value int value = ZKClientConfig.CLIENT_MAX_PACKET_LENGTH_DEFAULT; - conf.setProperty(ZKConfig.JUTE_MAXBUFFER, Integer.toString(value)); - result = conf.getInt(ZKConfig.JUTE_MAXBUFFER, defaultValue); + conf.setProperty(JUTE_MAXBUFFER, Integer.toString(value)); + result = conf.getInt(JUTE_MAXBUFFER, defaultValue); assertEquals(value, result); // property is set but with white spaces value = 12345; - conf.setProperty(ZKConfig.JUTE_MAXBUFFER, + conf.setProperty(JUTE_MAXBUFFER, " " + Integer.toString(value) + " "); - result = conf.getInt(ZKConfig.JUTE_MAXBUFFER, defaultValue); + result = conf.getInt(JUTE_MAXBUFFER, defaultValue); assertEquals(value, result); } diff --git a/src/java/test/org/apache/zookeeper/server/quorum/ReconfigDuringLeaderSyncTest.java b/src/java/test/org/apache/zookeeper/server/quorum/ReconfigDuringLeaderSyncTest.java index f7f0c7ce90a..d7299c03c8b 100644 --- a/src/java/test/org/apache/zookeeper/server/quorum/ReconfigDuringLeaderSyncTest.java +++ b/src/java/test/org/apache/zookeeper/server/quorum/ReconfigDuringLeaderSyncTest.java @@ -17,10 +17,6 @@ */ package org.apache.zookeeper.server.quorum; -import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; - import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; @@ -35,6 +31,7 @@ import org.apache.zookeeper.server.admin.AdminServer.AdminServerException; import org.apache.zookeeper.server.persistence.FileTxnSnapLog; import org.apache.zookeeper.server.quorum.flexible.QuorumMaj; +import org.apache.zookeeper.server.quorum.util.QuorumSocketFactory; import org.apache.zookeeper.test.ClientBase; import org.apache.zookeeper.test.ClientBase.CountdownWatcher; import org.junit.After; @@ -44,6 +41,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; + public class ReconfigDuringLeaderSyncTest extends QuorumPeerTestBase { protected static final Logger LOG = LoggerFactory.getLogger(ReconfigDuringLeaderSyncTest.class); private static int SERVER_COUNT = 3; @@ -211,7 +212,8 @@ public CustomQuorumPeer(Map quorumPeers, File snapDir, File int electionAlg, long myid, int tickTime, int initLimit, int syncLimit) throws IOException { super(quorumPeers, snapDir, logDir, electionAlg, myid, tickTime, initLimit, syncLimit, false, - ServerCnxnFactory.createFactory(new InetSocketAddress(clientPort), -1), new QuorumMaj(quorumPeers)); + ServerCnxnFactory.createFactory(new InetSocketAddress(clientPort), -1), + QuorumSocketFactory.createWithoutSSL(), new QuorumMaj(quorumPeers)); } /** From 30450268282d307e04c684dbcb0abb9c9bc59dca Mon Sep 17 00:00:00 2001 From: Powell Molleti Date: Mon, 6 Mar 2017 20:25:29 -0800 Subject: [PATCH 12/12] Add README to help explain what this code tries to accomplish. --- README_SSL.md | 149 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 README_SSL.md diff --git a/README_SSL.md b/README_SSL.md new file mode 100644 index 00000000000..948e7f9a31b --- /dev/null +++ b/README_SSL.md @@ -0,0 +1,149 @@ +# Zookeeper Dynamic Quorum SSL + +Provides SSL for Leader Election and ZAB i.e ports 3888 and 2888. + +Goal of this patch is to build on top of SSL changes for [branch-3.4](https://github.com/geek101/zookeeper/blob/branch-3.4/README_SSL.md) and in +the spirit of branch-3.5 provide support for dynamic reconfiguration, i.e do +not violate safety and liveliness even when SSL is enabled. + +### TODO + +* CA based cert verification currently has no support for changing CA in a +fault-tolerant way. Alternative to changing CA is to revoke the quorum peer +to be removed via CRL(s) hence this needs more discussion/debate perhaps. + +* Needs test framework and/or test cases to verify new functionality, this is +probably a significant amount of work gating this patch among other things. + +### Self Signed Certs How To + +The idea here is to propagate the new member(s) certificate fingerprint(s) +via the secure channel to the quorum via the reconfig() API. + +Each quorum peer will have its self-signed cert finger print (typically like +a SHA-256 digest) embedded into its server string. + +``` +server.1=125.23.63.23:2780:2783:participant:cert:SHA-256-XXXX;2791 +``` + +We also have support for both plain and secure port for ZookeeperServer hence +the config has been extended to reflect that as follows. + +``` +server.1=125.23.63.23:2780:2783:participant:cert:SHA-256-XXXX;plain:2791; +secure:2891 +``` + +A reconfiguration operation would work by submitting the new server +config (or the new quorum list) to reconfig() API. + +This has been tested to work with the current state of the patch. + +### Some details + +* [X509Util](https://github.com/geek101/zookeeper/blob/branch-3.5-ssl-review5/src/java/main/org/apache/zookeeper/common/X509Util.java) +becomes first class citizen and [QuorumX509Util](https://github.com/geek101/zookeeper/blob/branch-3.5-ssl-review5/src/java/main/org/apache/zookeeper/server/quorum/util/QuorumX509Util.java) and [ServerX509Util](https://github.com/geek101/zookeeper/blob/branch-3.5-ssl-review5/src/java/main/org/apache/zookeeper/server/util/ServerX509Util.java) +extend it. +* [ZKConfig](https://github.com/geek101/zookeeper/blob/branch-3.5-ssl-review5/src/java/main/org/apache/zookeeper/common/ZKConfig.java) +becomes an abstract class and [QuorumSslConfig](https://github.com/geek101/zookeeper/blob/branch-3.5-ssl-review5/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java) and +[ZookeeperServerConfig](https://github.com/geek101/zookeeper/blob/branch-3.5-ssl-review5/src/java/main/org/apache/zookeeper/server/ZookeeperServerConfig.java) implement it. +* [QuorumServer](https://github.com/geek101/zookeeper/blob/branch-3.5-ssl-review5/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java#L278) gets the parsing code for the extra cert information and gets +help from new [SSLCertCfg](https://github.com/geek101/zookeeper/blob/branch-3.5-ssl-review5/src/java/main/org/apache/zookeeper/SSLCertCfg.java) class. Dynamic config generation is handled +transparently due to this. +* [ZKDynamicX509TrustManager](https://github.com/geek101/zookeeper/blob/branch-3.5-ssl-review5/src/java/main/org/apache/zookeeper/server/quorum/util/ZKDynamicX509TrustManager.java) handles the dynamic verification of certs and +gets help from QuorumPeer's new API, +[getQuorumServerFingerPrintByElectionAddress()](https://github.com/geek101/zookeeper/blob/branch-3.5-ssl-review5/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java#L1642) and +[getQuorumServerFingerPrintByCert()](https://github.com/geek101/zookeeper/blob/branch-3.5-ssl-review5/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java#L1661) +* Support for a Quorum peer to also be authenticated as a [ZK client](https://github.com/geek101/zookeeper/blob/branch-3.5-ssl-review5/src/java/main/org/apache/zookeeper/server/util/ServerX509Util.java#L62) (this +will be removed if it breaks security and or is not needed) + + +### How to Run + +**Depends on Java 1.7+** + +##### Building + + +``` +git checkout branch-3.5-ssl-review5 +ant jar +``` + +Args to enable SSL: +``` +-Dquorum.ssl.enabled="true" +-Dquorum.ssl.keyStore.location="" +-Dquorum.ssl.keyStore.password="" +-Dquorum.ssl.trustStore.location="" +-Dquorum.ssl.trustStore.password="" +-Dquorum.ssl.digest.algos="" +``` + +Example run command: +``` +java -Dquorum.ssl.enabled="true" -Dquorum.ssl.keyStore.location="node1.ks" -Dquorum.ssl.keyStore.password="CertPassword1" -Dquorum.ssl.trustStore.location="truststore.ks" -Dquorum.ssl.trustStore.password="StorePass" -Dquorum.ssl.digest.algos="SHA-256" -cp zookeeper.jar:lib/* org.apache.zookeeper.server.quorum.QuorumPeerMain zoo1.cfg +``` + +##### Note + +Keystore password must be the same as password used to store the private key of the node. +Keystore cannot have more than 1 key. +To help with debug please add -Djavax.net.debug=ssl:handshake. + +#### Cert generation and test helpers + +##### Generating Root CA and certs for Zookeeper nodes +Use the scripts and config files in *resources/* directory. + +###### Step 1 +To generate root CA: + +``` +$ cd resources/x509ca +$ resources/x509ca$ ../init.sh +``` + +> use defaults and enter yes to load root self-signed cert to truststore +> truststore is *resources/x509ca/java/truststore.ks* + +###### Step 2 + +Now generate certs for every node, ex: + +``` +$ resources/x509ca$ ../gencert.sh node1 +``` + +> you will be prompted for private key password, enter: CertPassword1 +> note: you can enter any password but remember to change the script to support that if you do so. +> keystore is *resouces/x509ca/node1.ks* +> Repeat Step 2 for as many nodes as you want. +> These will be used by step 3. + +###### Step 3 + +Running a three node zookeeper cluster + +Create three loopback interfaces 127.0.1.1, 127.0.1.2, 127.0.1.3 and run *start_quorum.sh* in *config/multi/* +``` +$ sudo ifconfig lo:1 127.0.1.1 netmask 255.255.255.0 +$ cd conf/multi/ +$ ./start_quorum.sh +``` + +> Verify the logs in */tmp/zookeeper/multi/node.log* and SSL debug data in +> *conf/multi/node.out* +> If logs look good then you could use zkCli.sh to test the cluster. + +``` +bin/zkCli.sh -server 127.0.1.1:2181 +``` + +##### Unit test + +Currently unit test expects keystore files to be available via absolute path. +Edit *src/java/test/org/apache/zookeeper/server/quorum/QuorumSocketFactoryTest.java* and point **PATH** to *resources/* directory. + +Also generate certs and keys in *resources/x509ca2* for the negative test to pass. \ No newline at end of file