Merge 55348bd into d19861a

CHANGELOG.rst

@@ -7,23 +7,35 @@ Rasa Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to `Semantic Versioning`_ starting with version 1.0.
 
-[Unreleased 1.3.9]
-^^^^^^^^^^^^^^^^^^
+[Unreleased 1.3.10]
+^^^^^^^^^^^^^^^^^^^
 
 Added
 -----
 
 Fixed
 -----
-- fixed the hanging HTTP call with ``ner_duckling_http`` pipeline
-- Fixed text processing of ``intent`` attribute inside ``CountVectorFeaturizer``.
 
 Changed
 -------
 
 Removed
 -------
 
+[1.3.9] - 2019-10-10
+^^^^^^^^^^^^^^^^^^^^
+
+Added
+-----
+- Port of 1.2.10 (support for RabbitMQ TLS authentication and ``port`` key in
+  event broker endpoint config).
+- Port of 1.2.11 (support for passing a CA file for SSL certificate verification via the
+  --ssl-ca-file flag).
+
+Fixed
+-----
+- Fixed the hanging HTTP call with ``ner_duckling_http`` pipeline.
+- Fixed text processing of ``intent`` attribute inside ``CountVectorFeaturizer``.
 
 [1.3.8] - 2019-10-08
 ^^^^^^^^^^^^^^^^^^^^
@@ -84,6 +96,9 @@ Fixed
   for every given attribute. Non-content-bearing samples are converted to empty
   ``Doc``-objects. The resulting lists are merged with their preserved order and
   properly returned.
+- asyncio warnings are now only printed if the callback takes more than 100ms
+  (up from 1ms).
+- ``agent.load_model_from_server`` no longer affects logging.
 
 Changed
 -------
@@ -93,15 +108,6 @@ Changed
   (default behavior) or in a temporary directory by specifying the
   ``save_to_default_model_directory`` field in the training request.
 
-[1.3.4]
-^^^^^^^^^^^^^^^^^^^^
-
-Fixed
------
-- asyncio warnings are now only printed if the callback takes more than 100ms
-  (up from 1ms)
-- ``agent.load_model_from_server`` no longer affects logging
-
 [1.3.3] - 2019-09-13
 ^^^^^^^^^^^^^^^^^^^^
 
@@ -220,6 +226,28 @@ Removed
 -------
 - Removed ``--report`` argument from ``rasa test nlu``. All output files are stored in the ``--out`` directory.
 
+[1.2.11] - 2019-10-09
+^^^^^^^^^^^^^^^^^^^^^
+
+Added
+-----
+- Support for passing a CA file for SSL certificate verification via the
+  --ssl-ca-file flag
+
+[1.2.10] - 2019-10-08
+^^^^^^^^^^^^^^^^^^^^^
+
+Added
+-----
+- Added support for RabbitMQ TLS authentication. The following environment variables
+  need to be set:
+  ``RABBITMQ_SSL_CLIENT_CERTIFICATE`` - path to the SSL client certificate (required)
+  ``RABBITMQ_SSL_CLIENT_KEY`` - path to the SSL client key (required)
+  ``RABBITMQ_SSL_CA_FILE`` - path to the SSL CA file (optional, for certificate
+  verification)
+  ``RABBITMQ_SSL_KEY_PASSWORD`` - SSL private key password (optional)
+- Added ability to define the RabbitMQ port using the ``port`` key in the
+  ``event_broker`` endpoint config.
 
 [1.2.9] - 2019-09-17
 ^^^^^^^^^^^^^^^^^^^^

rasa/cli/arguments/run.py

@@ -67,6 +67,11 @@ def add_server_arguments(parser: argparse.ArgumentParser):
     server_arguments.add_argument(
         "--ssl-keyfile", help="Set the SSL Keyfile to create a TLS secured server."
     )
+    server_arguments.add_argument(
+        "--ssl-ca-file",
+        help="If your SSL certificate needs to be verified, you can specify the CA file "
+        "using this parameter.",
+    )
     server_arguments.add_argument(
         "--ssl-password",
         help="If your ssl-keyfile is protected by a password, you can specify it "

rasa/cli/x.py

@@ -81,6 +81,7 @@ def _rasa_service(
         jwt_method=args.jwt_method,
         ssl_certificate=args.ssl_certificate,
         ssl_keyfile=args.ssl_keyfile,
+        ssl_ca_file=args.ssl_ca_file,
         ssl_password=args.ssl_password,
     )
 

rasa/core/brokers/pika.py

@@ -5,6 +5,7 @@
 
 import time
 
+import rasa.core.brokers.utils as rasa_broker_utils
 from rasa.core.brokers.event_channel import EventChannel
 from rasa.utils.endpoints import EndpointConfig
 
@@ -18,6 +19,7 @@ def initialise_pika_connection(
     host: Text,
     username: Text,
     password: Text,
+    port: Union[Text, int] = 5672,
     connection_attempts: int = 20,
     retry_delay_in_seconds: Union[int, float] = 5,
 ) -> "BlockingConnection":
@@ -27,11 +29,13 @@ def initialise_pika_connection(
         host: Pika host
         username: username for authentication with Pika host
         password: password for authentication with Pika host
+        port: port of the Pika host
         connection_attempts: number of channel attempts before giving up
         retry_delay_in_seconds: delay in seconds between channel attempts
 
     Returns:
         Pika `BlockingConnection` with provided parameters
+
     """
 
     import pika
@@ -47,12 +51,14 @@ def initialise_pika_connection(
         # host seems to be just the host, so we use our parameters
         parameters = pika.ConnectionParameters(
             host,
+            port=port,
             credentials=pika.PlainCredentials(username, password),
             connection_attempts=connection_attempts,
             # Wait between retries since
             # it can take some time until
             # RabbitMQ comes up.
             retry_delay=retry_delay_in_seconds,
+            ssl_options=rasa_broker_utils.create_rabbitmq_ssl_options(host),
         )
     return pika.BlockingConnection(parameters)
 
@@ -62,6 +68,7 @@ def initialise_pika_channel(
     queue: Text,
     username: Text,
     password: Text,
+    port: Union[Text, int] = 5672,
     connection_attempts: int = 20,
     retry_delay_in_seconds: Union[int, float] = 5,
 ) -> "BlockingChannel":
@@ -72,15 +79,17 @@ def initialise_pika_channel(
         queue: Pika queue to declare
         username: username for authentication with Pika host
         password: password for authentication with Pika host
+        port: port of the Pika host
         connection_attempts: number of channel attempts before giving up
         retry_delay_in_seconds: delay in seconds between channel attempts
 
     Returns:
         Pika `BlockingChannel` with declared queue
+
     """
 
     connection = initialise_pika_connection(
-        host, username, password, connection_attempts, retry_delay_in_seconds
+        host, username, password, port, connection_attempts, retry_delay_in_seconds
     )
 
     return _declare_pika_channel_with_queue(connection, queue)
@@ -127,6 +136,7 @@ def __init__(
         host: Text,
         username: Text,
         password: Text,
+        port: Union[int, Text] = 5672,
         queue: Text = "rasa_core_events",
         loglevel: Union[Text, int] = logging.WARNING,
     ):
@@ -136,6 +146,7 @@ def __init__(
         self.host = host
         self.username = username
         self.password = password
+        self.port = port
         self.channel = None  # delay opening channel until first event
 
     def __del__(self) -> None:
@@ -153,6 +164,7 @@ def _open_channel(
             self.queue,
             self.username,
             self.password,
+            self.port,
             connection_attempts,
             retry_delay_in_seconds,
         )

rasa/core/brokers/utils.py

@@ -1,12 +1,14 @@
 import logging
+import os
 import typing
-from typing import Optional
+from typing import Optional, Text
 
 import rasa.utils.common as rasa_utils
 from rasa.utils.endpoints import EndpointConfig
 
 if typing.TYPE_CHECKING:
     from rasa.core.brokers.event_channel import EventChannel
+    import pika
 
 logger = logging.getLogger(__name__)
 
@@ -52,3 +54,50 @@ def load_event_channel_from_module_string(
             "Not using any event channel. Error: {}".format(broker_config.type, e)
         )
         return None
+
+
+def create_rabbitmq_ssl_options(
+    rabbitmq_host: Optional[Text] = None
+) -> Optional["pika.SSLOptions"]:
+    """Create RabbitMQ SSL options.
+
+    Requires the following environment variables to be set:
+
+        RABBITMQ_SSL_CLIENT_CERTIFICATE - path to the SSL client certificate (required)
+        RABBITMQ_SSL_CLIENT_KEY - path to the SSL client key (required)
+        RABBITMQ_SSL_CA_FILE - path to the SSL CA file for verification (optional)
+        RABBITMQ_SSL_KEY_PASSWORD - SSL private key password (optional)
+
+    Details on how to enable RabbitMQ TLS support can be found here:
+    https://www.rabbitmq.com/ssl.html#enabling-tls
+
+    Args:
+        rabbitmq_host: RabbitMQ hostname
+
+    Returns:
+        Pika SSL context of type `pika.SSLOptions` if
+        the RABBITMQ_SSL_CLIENT_CERTIFICATE and RABBITMQ_SSL_CLIENT_KEY
+        environment variables are valid paths, else `None`.
+
+    """
+
+    client_certificate_path = os.environ.get("RABBITMQ_SSL_CLIENT_CERTIFICATE")
+    client_key_path = os.environ.get("RABBITMQ_SSL_CLIENT_KEY")
+
+    if client_certificate_path and client_key_path:
+        import pika
+        import rasa.server
+
+        logger.debug(
+            "Configuring SSL context for RabbitMQ host '{}'.".format(rabbitmq_host)
+        )
+
+        ca_file_path = os.environ.get("RABBITMQ_SSL_CA_FILE")
+        key_password = os.environ.get("RABBITMQ_SSL_KEY_PASSWORD")
+
+        ssl_context = rasa.server.create_ssl_context(
+            client_certificate_path, client_key_path, ca_file_path, key_password
+        )
+        return pika.SSLOptions(ssl_context, rabbitmq_host)
+    else:
+        return None

rasa/core/run.py

@@ -150,6 +150,7 @@ def serve_application(
     log_file: Optional[Text] = None,
     ssl_certificate: Optional[Text] = None,
     ssl_keyfile: Optional[Text] = None,
+    ssl_ca_file: Optional[Text] = None,
     ssl_password: Optional[Text] = None,
 ):
     from rasa import server
@@ -171,7 +172,9 @@ def serve_application(
         log_file=log_file,
     )
 
-    ssl_context = server.create_ssl_context(ssl_certificate, ssl_keyfile, ssl_password)
+    ssl_context = server.create_ssl_context(
+        ssl_certificate, ssl_keyfile, ssl_ca_file, ssl_password
+    )
     protocol = "https" if ssl_context else "http"
 
     logger.info(

rasa/server.py

@@ -1,12 +1,12 @@
 import logging
+import multiprocessing
 import os
 import tempfile
 import traceback
-import multiprocessing
+import typing
 from functools import wraps, reduce
 from inspect import isawaitable
 from typing import Any, Callable, List, Optional, Text, Union
-from ssl import SSLContext
 
 from sanic import Sanic, response
 from sanic.request import Request
@@ -19,7 +19,6 @@
 import rasa.utils.common
 import rasa.utils.endpoints
 import rasa.utils.io
-
 from rasa import model
 from rasa.constants import (
     MINIMUM_COMPATIBLE_VERSION,
@@ -44,6 +43,9 @@
 from rasa.nlu.test import run_evaluation
 from rasa.utils.endpoints import EndpointConfig
 
+if typing.TYPE_CHECKING:
+    from ssl import SSLContext
+
 logger = logging.getLogger(__name__)
 
 OUTPUT_CHANNEL_QUERY_KEY = "output_channel"
@@ -226,14 +228,28 @@ async def authenticate(request: Request):
 def create_ssl_context(
     ssl_certificate: Optional[Text],
     ssl_keyfile: Optional[Text],
-    ssl_password: Optional[Text],
-) -> Optional[SSLContext]:
-    """Create a SSL context (for the sanic server) if a proper certificate is passed."""
+    ssl_ca_file: Optional[Text] = None,
+    ssl_password: Optional[Text] = None,
+) -> Optional["SSLContext"]:
+    """Create an SSL context if a proper certificate is passed.
+
+    Args:
+        ssl_certificate: path to the SSL client certificate
+        ssl_keyfile: path to the SSL key file
+        ssl_ca_file: path to the SSL CA file for verification (optional)
+        ssl_password: SSL private key password (optional)
+
+    Returns:
+        SSL context if a valid certificate chain can be loaded, `None` otherwise.
+
+    """
 
     if ssl_certificate:
         import ssl
 
-        ssl_context = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH)
+        ssl_context = ssl.create_default_context(
+            purpose=ssl.Purpose.CLIENT_AUTH, cafile=ssl_ca_file
+        )
         ssl_context.load_cert_chain(
             ssl_certificate, keyfile=ssl_keyfile, password=ssl_password
         )

rasa/version.py

@@ -1 +1 @@
-__version__ = "1.3.8"
+__version__ = "1.3.9"

tests/cli/test_rasa_run.py

@@ -20,9 +20,10 @@ def test_run_help(run):
                 [--cors [CORS [CORS ...]]] [--enable-api]
                 [--remote-storage REMOTE_STORAGE]
                 [--ssl-certificate SSL_CERTIFICATE]
-                [--ssl-keyfile SSL_KEYFILE] [--ssl-password SSL_PASSWORD]
-                [--credentials CREDENTIALS] [--connector CONNECTOR]
-                [--jwt-secret JWT_SECRET] [--jwt-method JWT_METHOD]
+                [--ssl-keyfile SSL_KEYFILE] [--ssl-ca-file SSL_CA_FILE]
+                [--ssl-password SSL_PASSWORD] [--credentials CREDENTIALS]
+                [--connector CONNECTOR] [--jwt-secret JWT_SECRET]
+                [--jwt-method JWT_METHOD]
                 {actions} ... [model-as-positional-argument]"""
 
     lines = help_text.split("\n")

tests/cli/test_rasa_shell.py

@@ -6,9 +6,10 @@ def test_shell_help(run):
                   [--cors [CORS [CORS ...]]] [--enable-api]
                   [--remote-storage REMOTE_STORAGE]
                   [--ssl-certificate SSL_CERTIFICATE]
-                  [--ssl-keyfile SSL_KEYFILE] [--ssl-password SSL_PASSWORD]
-                  [--credentials CREDENTIALS] [--connector CONNECTOR]
-                  [--jwt-secret JWT_SECRET] [--jwt-method JWT_METHOD]
+                  [--ssl-keyfile SSL_KEYFILE] [--ssl-ca-file SSL_CA_FILE]
+                  [--ssl-password SSL_PASSWORD] [--credentials CREDENTIALS]
+                  [--connector CONNECTOR] [--jwt-secret JWT_SECRET]
+                  [--jwt-method JWT_METHOD]
                   {nlu} ... [model-as-positional-argument]"""
 
     lines = help_text.split("\n")