Switch driver to confluent-kafka client library

This patch switches the kafka python client from kafka-python to
confluent-kafka due to documented threading issues with the
kafka-python consumer and the recommendation to use multiplrocessing.
The confluent-kafka client leverages the high performance librdkafka
C client and is safe for multiple thread use.

This patch:
* switches to confluent-kafka library
* revises consumer and producer message operations
* utilizes event.tpool method for confluent-kafka blocking calls
* updates unit tests
* adds kafka specific timeouts for functional tests
* adds release note

Depends-On: Ice374dca539b8ed1b1965b75379bad5140121483
Change-Id: Idfb9fe3700d882c8285c6dc56b0620951178eba2

.zuul.yaml

@@ -77,7 +77,7 @@
       devstack_plugins:
         devstack-plugin-amqp1: git://git.openstack.org/openstack/devstack-plugin-amqp1
       zuul_copy_output:
-        '{{ devstack_base_dir }}/logs/qdrouterd.log': logs
+        '{{ devstack_log_dir }}/qdrouterd.log': logs
 
 
 - job:
@@ -102,8 +102,7 @@
       devstack_plugins:
         devstack-plugin-kafka: git://git.openstack.org/openstack/devstack-plugin-kafka
       zuul_copy_output:
-        '{{ devstack_base_dir }}/logs/qdrouterd.log': logs
-
+        '{{ devstack_log_dir }}/server.log': logs
 
 - job:
     name: oslo.messaging-src-dsvm-full-kafka-centos-7

bindep.txt

@@ -32,6 +32,8 @@ swig [platform:rpm amqp1]
 
 # kafka dpkg
 openjdk-8-jdk [platform:dpkg kafka]
+librdkafka1 [platform:dpkg kafka]
 
 # kafka rpm
 java-1.8.0-openjdk [platform:rpm kafka]
+librdkafka [platform:rpm kafka]

doc/requirements.txt

@@ -8,6 +8,6 @@ reno>=2.5.0 # Apache-2.0
 
 # imported when the source code is parsed for generating documentation:
 fixtures>=3.0.0 # Apache-2.0/BSD
-kafka-python>=1.3.1 # Apache-2.0
+confluent-kafka>=0.11.6 # Apache-2.0
 pyngus>=2.2.0 # Apache-2.0
 tenacity>=3.2.1 # Apache-2.0

lower-constraints.txt

@@ -7,6 +7,7 @@ cachetools==2.0.0
 cffi==1.7.0
 cliff==2.8.0
 cmd2==0.8.0
+confluent-kafka==0.11.6
 contextlib2==0.4.0
 coverage==4.0
 debtcollector==1.2.0
@@ -26,7 +27,6 @@ hacking==0.12.0
 imagesize==0.7.1
 iso8601==0.1.11
 Jinja2==2.10
-kafka-python==1.3.1
 keystoneauth1==3.4.0
 kombu==4.0.0
 linecache2==1.0.0