ZOOKEEPER-3567: add SSL support for zkpython

zookeeper-contrib/build-contrib.xml

@@ -32,7 +32,7 @@
 
   <property name="lib.dir"  location="${zk.root}/zookeeper-server/src/main/resources/lib"/>
 
-  <property name="build.dir" location="${zk.root}/build/zookeeper-contrib/zookeeper-contrib-${name}"/>
+  <property name="build.dir" location="${zk.root}/zookeeper-contrib/zookeeper-contrib-${name}/target"/>
   <property name="build.classes" location="${build.dir}/classes"/>
   <property name="build.test" location="${build.dir}/test"/>
 
@@ -179,7 +179,7 @@
       <!-- we can't use id=classpath, because available fails if fileset directory
            doesn't exist -->
       <classpath>
-        <pathelement location="${zk.root}/build/classes"/>
+        <pathelement location="${zk.root}/zookeeper-server/target/classes"/>
       </classpath>
     </available>
   </target>
@@ -190,7 +190,7 @@
 
 
   <target name="checkMainTestIsAvailable">
-    <available file="${zk.root}/build/test/classes/org/apache/zookeeper/test/ClientBase.class"
+    <available file="${zk.root}/zookeeper-server/target/test-classes/org/apache/zookeeper/test/ClientBase.class"
                property="mainTestIsCompiled">
     </available>
   </target>

zookeeper-contrib/zookeeper-contrib-zkpython/README

@@ -7,14 +7,18 @@ DEPENDENCIES:
 
 This has only been tested against SVN (i.e. 3.2.0 in development) but should work against 3.1.1. 
 
-You will need the Python development headers installed to build the module - on many package-management systems, these can be found in python-devel.
+You will need the Python development headers installed to build the module - on many package-management systems, these can be found in python-devel. (On ubuntu 18.4, install python2.7 and python2.7-dev.)
 
 Python >= 2.6 is required. We have tested against 2.6. We have not tested against 3.x. 
 
+E.g. setting up tpyhon and python devel on ubuntu 18.4:
+sudo apt-get install python2.7 python2.7-dev
+sudo update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1
+
 BUILD AND INSTALL:
 -------------------
 
-To install, make sure that the C client has been built and that the libraries are installed in /usr/local/lib (or change this directory in setup.py). Then run:
+To install, make sure that the C client has been built (use `mvn clean install -DskipTests -Pfull-build` in the root folder of zookeeper) or that the zookeeper C libraries are installed in /usr/local/lib (or change this directory in setup.py). Then run:
 
 ant install
 

zookeeper-contrib/zookeeper-contrib-zkpython/build.xml

@@ -20,10 +20,10 @@
 <project name="zkpython" default="install">
   <import file="../build-contrib.xml"/>
   <property name="python.src.dir" value="src/python"/>
-  <property name="test.build.dir" value="build/test/" />
   <property name="test.src.dir" value="src/test"/>
   <property name="test.log.dir" value="${build.test}/logs" />
   <property name="test.output" value="no" />
+  <property name="test.output" value="no" />
   <property name="test.timeout" value="900000" />
 
   <target name="test"

zookeeper-contrib/zookeeper-contrib-zkpython/src/c/pyzk_docstrings.h

@@ -568,6 +568,33 @@ static const char pyzk_init_doc[] =
 " an integer handle. If it fails to create \n"
 " a new zhandle the function throws an exception.\n";
 
+static const char pyzk_init_ssl_doc[] =
+"This method creates a new handle and a zookeeper SSL session that corresponds\n"
+"to that handle. Session establishment is asynchronous, meaning that the\n"
+"session should not be considered established until (and unless) an\n"
+"event of state CONNECTED_STATE is received.\n"
+"PARAMETERS:\n"
+" host: comma separated host:port pairs, each corresponding to a zk\n"
+"  server. e.g. '127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002'\n"
+" cert_str: SSL certificate string e.g. 'server.cert,client.cert,client-priv-key.pom,passwd'\n"
+"\n"
+"(subsequent parameters are optional)\n"
+" fn: the global watcher callback function. When notifications are\n"
+"  triggered this function will be invoked.\n"
+" recv_timeout: \n"
+" (clientid, passwd)\n"
+" clientid the id of a previously established session that this\n"
+"  client will be reconnecting to. Clients can access the session id of an established, valid,\n"
+"  connection by calling zoo_client_id. If\n"
+"  the specified clientid has expired, or if the clientid is invalid for \n"
+"  any reason, the returned zhandle_t will be invalid -- the zhandle_t \n"
+"  state will indicate the reason for failure (typically\n"
+"  EXPIRED_SESSION_STATE).\n"
+"\n"
+"RETURNS:\n"
+" an integer handle. If it fails to create \n"
+" a new zhandle the function throws an exception.\n";
+
 static const char pyzk_get_doc[] = 
 " gets the data associated with a node synchronously.\n"
 "\n"

zookeeper-contrib/zookeeper-contrib-zkpython/src/c/zookeeper.c

@@ -599,12 +599,13 @@ void acl_completion_dispatch(int rc, struct ACL_vector *acl, struct Stat *stat,
 /* ZOOKEEPER API IMPLEMENTATION */
 /* -------------------------------------------------------------------------- */
 
-static PyObject *pyzookeeper_init(PyObject *self, PyObject *args)
-{
+
+static PyObject *pyzookeeper_init_optional_ssl(PyObject *self, PyObject *args, int ssl) {
   const char *host;
+  const char *cert_str;
   PyObject *watcherfn = Py_None;
+  zhandle_t *zh = NULL;
   int recv_timeout = 10000;
-  //  int clientid = -1;
   clientid_t cid;
   cid.client_id = -1;
   const char *passwd;
@@ -621,9 +622,14 @@ static PyObject *pyzookeeper_init(PyObject *self, PyObject *args)
     return NULL;
   }
 
-  if (!PyArg_ParseTuple(args, "s|Oi(Ls)", &host, &watcherfn, &recv_timeout, &cid.client_id, &passwd)) 
-    return NULL;
-
+  if (ssl) {
+    if (!PyArg_ParseTuple(args, "ss|Oi(Ls)", &host, &cert_str, &watcherfn, &recv_timeout, &cid.client_id, &passwd))
+      return NULL;
+  } else {
+    if (!PyArg_ParseTuple(args, "s|Oi(Ls)", &host, &watcherfn, &recv_timeout, &cid.client_id, &passwd))
+        return NULL;
+  }
+
   if (cid.client_id != -1) {
     strncpy(cid.passwd, passwd, 16*sizeof(char));
   }
@@ -635,21 +641,36 @@ static PyObject *pyzookeeper_init(PyObject *self, PyObject *args)
     }
   }
   watchers[handle] = pyw;
-  zhandle_t *zh = zookeeper_init( host, watcherfn != Py_None ? watcher_dispatch : NULL, 
-                                  recv_timeout, cid.client_id == -1 ? 0 : &cid, 
-                                  pyw,
-                                  0 ); 
+
+  if (ssl) {
+    zh = zookeeper_init_ssl( host, cert_str, watcherfn != Py_None ? watcher_dispatch : NULL,
+                             recv_timeout, cid.client_id == -1 ? 0 : &cid, pyw, 0 );
+  } else {
+    zh = zookeeper_init( host, watcherfn != Py_None ? watcher_dispatch : NULL,
+                         recv_timeout, cid.client_id == -1 ? 0 : &cid, pyw, 0 );
+  }
 
   if (zh == NULL)
     {
-      PyErr_SetString( ZooKeeperException, "Could not internally obtain zookeeper handle" );
+      PyErr_SetString( ZooKeeperException, "Could not internally obtain SSL zookeeper handle" );
       return NULL;
     }
 
   zhandles[handle] = zh;
   return Py_BuildValue( "i", handle);
 }
 
+static PyObject *pyzookeeper_init(PyObject *self, PyObject *args)
+{
+  return pyzookeeper_init_optional_ssl(self, args, 0);
+}
+
+
+static PyObject *pyzookeeper_init_ssl(PyObject *self, PyObject *args)
+{
+  return pyzookeeper_init_optional_ssl(self, args, 1);
+}
+
 
 /* -------------------------------------------------------------------------- */
 /* Asynchronous API implementation */
@@ -1497,6 +1518,7 @@ PyObject *pyzoo_deterministic_conn_order(PyObject *self, PyObject *args)
 
 static PyMethodDef ZooKeeperMethods[] = {
   {"init", pyzookeeper_init, METH_VARARGS, pyzk_init_doc },
+  {"init_ssl", pyzookeeper_init_ssl, METH_VARARGS, pyzk_init_ssl_doc },
   {"create",pyzoo_create, METH_VARARGS, pyzk_create_doc },
   {"delete",pyzoo_delete, METH_VARARGS, pyzk_delete_doc },
   {"get_children", pyzoo_get_children, METH_VARARGS, pyzk_get_children_doc },

zookeeper-contrib/zookeeper-contrib-zkpython/src/python/setup.py

@@ -21,11 +21,11 @@
 zookeepermodule = Extension("zookeeper",
                             sources=["src/c/zookeeper.c"],
                             include_dirs=[zookeeper_basedir + "/zookeeper-client/zookeeper-client-c/include",
-                                          zookeeper_basedir + "/build/c",
+                                          zookeeper_basedir + "/zookeeper-client/zookeeper-client-c/target/c",
                                           zookeeper_basedir + "/zookeeper-client/zookeeper-client-c/generated"],
                             libraries=["zookeeper_mt"],
                             library_dirs=[zookeeper_basedir + "/zookeeper-client/zookeeper-client-c/.libs/",
-                                          zookeeper_basedir + "/build/c/.libs/",
+                                          zookeeper_basedir + "/zookeeper-client/zookeeper-client-c/target/c/.libs/",
                                           zookeeper_basedir + "/build/test/test-cppunit/.libs",
                                           "/usr/local/lib"
                                           ])

zookeeper-contrib/zookeeper-contrib-zkpython/src/test/connection_test.py

@@ -58,6 +58,37 @@ def connection_watcher(handle, type, state, path):
                           self.handle,
                           "/")
 
+
+    def testsslconnection(self):
+        cv = threading.Condition()
+        self.connected = False
+        def connection_watcher(handle, type, state, path):
+            cv.acquire()
+            self.connected = True
+            self.assertEqual(zookeeper.CONNECTED_STATE, state)
+            self.handle = handle
+            cv.notify()
+            cv.release()
+
+        cv.acquire()
+        ret = zookeeper.init_ssl(self.sslhost, self.sslcert, connection_watcher)
+        cv.wait(15.0)
+        cv.release()
+        self.assertEqual(self.connected, True, "SSL Connection timed out to " + self.host)
+        self.assertEqual(zookeeper.CONNECTED_STATE, zookeeper.state(self.handle))
+
+        self.assertEqual(zookeeper.close(self.handle), zookeeper.OK)
+        # Trying to close the same handle twice is an error, and the C library will segfault on it
+        # so make sure this is caught at the Python module layer
+        self.assertRaises(zookeeper.ZooKeeperException,
+                          zookeeper.close,
+                          self.handle)
+
+        self.assertRaises(zookeeper.ZooKeeperException,
+                          zookeeper.get,
+                          self.handle,
+                          "/")
+
     def testhandlereuse(self):
         """
         Test a) multiple concurrent connections b) reuse of closed handles

zookeeper-contrib/zookeeper-contrib-zkpython/src/test/gencerts.sh

@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This script cleans up old transaction logs and snapshots
+#
+
+#
+# If this scripted is run out of /usr/bin or some other system bin directory
+# it should be linked to and not copied. Things like java jar files are found
+# relative to the canonical path of this script.
+#
+
+# Generate the root key
+openssl genrsa -out rootkey.pem 2048
+
+#Generate the root Cert
+openssl req -x509 -new -key rootkey.pem -out root.crt -config <(
+cat <<-EOF
+[ req ]
+default_bits = 2048
+prompt = no
+default_md = sha256
+distinguished_name = dn
+
+[ dn ]
+C = US
+ST = California
+L = San Francisco
+O = Bookkeeper
+emailAddress = dev@bookkeeper.apache.org
+CN = bookkeeper.apache.org
+EOF
+)
+
+#Generate Client Key
+openssl genrsa -out clientkey.pem 2048
+
+#Generate Client Cert
+openssl req -new -key clientkey.pem -out client.csr -config <(
+cat <<-EOF
+[ req ]
+default_bits = 2048
+prompt = no
+default_md = sha256
+distinguished_name = dn
+
+[ dn ]
+C = US
+ST = California
+L = San Francisco
+O = Bookkeeper
+emailAddress = dev@bookkeeper.apache.org
+CN = bookkeeper.apache.org
+EOF
+)
+openssl x509 -req -in client.csr -CA root.crt -CAkey rootkey.pem -CAcreateserial -days 3650 -out client.crt
+
+#Export in pkcs12 format
+openssl pkcs12 -export -in client.crt -inkey clientkey.pem -out client.pkcs12 -password pass:password
+
+# Import Keystore in JKS
+keytool -importkeystore -srckeystore client.pkcs12 -destkeystore client.jks -srcstoretype pkcs12 -srcstorepass password -deststorepass password
+
+############################################################
+
+#Generate Server key
+openssl genrsa -out serverkey.pem 2048
+
+#Generate Server Cert
+openssl req -new -key serverkey.pem -out server.csr -config <(
+cat <<-EOF
+[ req ]
+default_bits = 2048
+prompt = no
+default_md = sha256
+distinguished_name = dn
+
+[ dn ]
+C = US
+ST = California
+L = San Francisco
+O = Bookkeeper
+emailAddress = dev@bookkeeper.apache.org
+CN = bookkeeper.apache.org
+EOF
+)
+openssl x509 -req -in server.csr -CA root.crt -CAkey rootkey.pem -CAcreateserial -days 3650 -out server.crt
+
+#Export in pkcs12 format
+openssl pkcs12 -export -in server.crt -inkey serverkey.pem -out server.pkcs12 -password pass:password
+
+# Import Keystore in JKS
+keytool -importkeystore -srckeystore server.pkcs12 -destkeystore server.jks -srcstoretype pkcs12 -srcstorepass password -deststorepass password
+
+
+keytool -importcert -keystore server.jks -file root.crt -storepass password -noprompt
+
+keytool -importcert -alias ca -file root.crt -keystore clienttrust.jks -storepass password -noprompt
+
+keytool -importcert -alias clientcert -file client.crt -keystore clienttrust.jks -storepass password -noprompt
+
+keytool -importcert -alias ca -file root.crt -keystore servertrust.jks -storepass password -noprompt
+keytool -importcert -alias servercert -file server.crt -keystore servertrust.jks -storepass password -noprompt

zookeeper-contrib/zookeeper-contrib-zkpython/src/test/run_tests.sh

@@ -30,11 +30,12 @@ else
 fi
 
 # Find the build directory containing zookeeper.so
-SO_PATH=`find ../../build/ -name "zookeeper.so" | head -1`
+SO_PATH=`find ./target/ -name "zookeeper.so" | head -1`
 PYTHONPATH=`dirname $SO_PATH`
-LIB_PATH=../../build/c/.libs/:../../build/test/test-cppunit/.libs
+LIB_PATH=../../zookeeper-client/zookeeper-client-c/target/c/.libs
 for test in `ls $1/*_test.py`; 
 do
     echo "Running $test"
+    echo "Running LD_LIBRARY_PATH=$LIB_PATH:$LD_LIBRARY_PATH DYLD_LIBRARY_PATH=$LIB_PATH:$DYLD_LIBRARY_PATH PYTHONPATH=$PYTHONPATH python $test"
     LD_LIBRARY_PATH=$LIB_PATH:$LD_LIBRARY_PATH DYLD_LIBRARY_PATH=$LIB_PATH:$DYLD_LIBRARY_PATH PYTHONPATH=$PYTHONPATH python $test
 done