Merge tag 'vumi-0.6.18'

Version Bump 0.6.18
praekeltfoundation · Aug 28, 2017 · 767eac6 · 767eac6
2 parents dbf04ee + 3eac558
commit 767eac6
Show file tree

Hide file tree

Showing 20 changed files with 270 additions and 85 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -3,6 +3,7 @@ sudo: false # We want container builds.
 language: python
 python:
   - "2.7"
+  - "3.6"
 node_js:
   - "0.10"
 matrix:
@@ -19,7 +20,12 @@ matrix:
     # Test on pypy without coverage, because it's unnecessary and very slow.
     # Also, we hit an obscure GC bug in pypy<=2.6.0 so we need at least 2.6.1.
     - python: "pypy"
-      env: PYPY_VERSION="4.0.1" NO_COVERAGE=1
+      env: PYPY_VERSION="5.6.0" NO_COVERAGE=1
+    - python: "3.6"
+      env: RUN_TESTS="false"
+  allow_failures:
+    - python: "3.6"
+    - env: RUN_TESTS="true"
 
 cache:
   directories:
@@ -32,6 +38,7 @@ before_install:
   # Default values for our build envvars.
   - export RIAK_VERSION="${RIAK_VERSION-1.4.12}"
   - export TWISTED_VERSION="${TWISTED_VERSION-Twisted}"
+  - export RUN_TESTS="${RUN_TESTS-true}"
   # If necessary, set up an appropriate version of pypy.
   - if [ ! -z "$PYPY_VERSION" ]; then source utils/setup-pypy-travis.sh; fi
   - if [ ! -z "$PYPY_VERSION" ]; then python --version 2>&1 | fgrep "PyPy $PYPY_VERSION"; fi
@@ -52,7 +59,7 @@ before_script:
   - $HOME/riak/bin/riak-admin member-status
 script:
   - if [ -z "$NO_COVERAGE" ]; then COVERAGE_CMD="coverage run --source=vumi"; else COVERAGE_CMD=""; fi
-  - VUMI_TEST_ASSERT_CLOSED=true VUMI_TEST_TIMEOUT=20 VUMITEST_REDIS_DB=1 VUMI_TEST_NODE_PATH="$(which node)" $COVERAGE_CMD `which trial` vumi
+  - if [ "$RUN_TESTS" == "true" ]; then VUMI_TEST_ASSERT_CLOSED=true VUMI_TEST_TIMEOUT=20 VUMITEST_REDIS_DB=1 VUMI_TEST_NODE_PATH="$(which node)" $COVERAGE_CMD `which trial` vumi; fi
 
 after_success:
   - if [ -z "$NO_COVERAGE" ]; then coveralls; fi
diff --git a/README.rst b/README.rst
@@ -10,9 +10,9 @@ be fixed.
 
 Documentation available online at http://vumi.readthedocs.org/ and in the `docs` directory of the repository.
 
-|vumi-ver| |vumi-ci| |vumi-cover| |python-ver| |vumi-docs| |vumi-downloads| |vumi-license|
+|vumi-ver| |vumi-ci| |vumi-cover| |python-ver| |vumi-docs| |vumi-license|
 
-.. |vumi-ver| image:: https://pypip.in/v/vumi/badge.png?text=pypi
+.. |vumi-ver| image:: https://img.shields.io/pypi/v/vumi.svg
     :alt: Vumi version
     :scale: 100%
     :target: https://pypi.python.org/pypi/vumi
@@ -27,7 +27,7 @@ Documentation available online at http://vumi.readthedocs.org/ and in the `docs`
     :scale: 100%
     :target: https://coveralls.io/r/praekelt/vumi
 
-.. |python-ver| image:: https://pypip.in/py_versions/vumi/badge.svg
+.. |python-ver| image:: https://img.shields.io/pypi/pyversions/vumi.svg
     :alt: Python version
     :scale: 100%
     :target: https://pypi.python.org/pypi/vumi
@@ -37,12 +37,13 @@ Documentation available online at http://vumi.readthedocs.org/ and in the `docs`
     :scale: 100%
     :target: http://vumi.readthedocs.org/
 
-.. |vumi-downloads| image:: https://pypip.in/download/vumi/badge.svg
+.. This is commented out because downloads stats from PyPI are not working
+.. .. |vumi-downloads| image:: https://img.shields.io/pypi/dm/vumi.svg
     :alt: Vumi downloads from PyPI
     :scale: 100%
     :target: https://pypi.python.org/pypi/vumi
 
-.. |vumi-license| image:: https://pypip.in/license/vumi/badge.svg
+.. |vumi-license| image:: https://img.shields.io/pypi/l/vumi.svg
     :target: https://pypi.python.org/pypi/vumi
     :alt: Vumi license
 

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,7 +1,7 @@
 FROM praekeltfoundation/python-base
 MAINTAINER Praekelt Foundation <dev@praekeltfoundation.org>
 
-ENV VUMI_VERSION "0.6.13"
+ENV VUMI_VERSION "0.6.18"
 RUN pip install vumi==$VUMI_VERSION
 
 COPY ./vumi-entrypoint.sh /app/vumi-entrypoint.sh

diff --git a/docs/conf.py b/docs/conf.py
@@ -53,7 +53,7 @@
 # The short X.Y version.
 version = '0.6'
 # The full version, including alpha/beta/rc tags.
-release = '0.6.13'
+release = '0.6.18'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

diff --git a/setup.py b/setup.py
@@ -12,7 +12,7 @@
 
 setup(
     name="vumi",
-    version="0.6.13",
+    version="0.6.18",
     url='http://github.com/praekelt/vumi',
     license='BSD',
     description="Super-scalable messaging engine for the delivery of SMS, "
@@ -40,7 +40,7 @@
         'txAMQP>=0.6.2',
         'PyYAML',
         'iso8601',
-        'pyOpenSSL',
+        'pyOpenSSL>= 16.0.0',
         'certifi',
         'service_identity',
         'txssmi>=0.3.0',
@@ -50,7 +50,7 @@
         'python-smpp>=0.1.5',
         'pytz',
         'riak>=2.1',
-        'txJSON-RPC==0.3.1',
+        'txJSON-RPC==0.5',
         'txTwitter>=0.1.4a',
         'treq<16.12.0',
         'confmodel>=0.2.0',

diff --git a/utils/setup-pypy-travis.sh b/utils/setup-pypy-travis.sh
@@ -10,6 +10,7 @@ eval "$(pyenv init -)"
 eval "$(pyenv virtualenv-init -)"
 
 # Install pypy and make a virtualenv for it.
+pyenv install --list
 pyenv install -s pypy-$PYPY_VERSION
 pyenv global pypy-$PYPY_VERSION
 virtualenv -p $(which python) ~/env-pypy-$PYPY_VERSION

diff --git a/vumi/__init__.py b/vumi/__init__.py
@@ -2,4 +2,4 @@
 Vumi scalable text messaging engine.
 """
 
-__version__ = "0.6.13"
+__version__ = "0.6.18"
diff --git a/vumi/application/sandbox.py b/vumi/application/sandbox.py
@@ -26,7 +26,7 @@
 
 from OpenSSL.SSL import (
     VERIFY_PEER, VERIFY_FAIL_IF_NO_PEER_CERT, VERIFY_CLIENT_ONCE, VERIFY_NONE,
-    SSLv3_METHOD, SSLv23_METHOD, TLSv1_METHOD)
+    SSLv23_METHOD, TLSv1_METHOD)
 
 from vumi.config import ConfigText, ConfigInt, ConfigList, ConfigDict
 from vumi.application.base import ApplicationWorker
@@ -883,7 +883,6 @@ def _make_request_from_command(self, method, command):
             'VERIFY_FAIL_IF_NO_PEER_CERT': VERIFY_FAIL_IF_NO_PEER_CERT,
         }
         method_map = {
-            'SSLv3': SSLv3_METHOD,
             'SSLv23': SSLv23_METHOD,
             'TLSv1': TLSv1_METHOD,
         }

diff --git a/vumi/application/tests/test_sandbox.py b/vumi/application/tests/test_sandbox.py
@@ -12,8 +12,8 @@
 import warnings
 
 from OpenSSL.SSL import (
-    VERIFY_PEER, VERIFY_FAIL_IF_NO_PEER_CERT, VERIFY_NONE,
-    SSLv3_METHOD, SSLv23_METHOD, TLSv1_METHOD)
+    VERIFY_PEER, VERIFY_FAIL_IF_NO_PEER_CERT, VERIFY_NONE, SSLv23_METHOD,
+    TLSv1_METHOD)
 
 from twisted.internet.defer import (
     inlineCallbacks, fail, succeed, DeferredQueue)
@@ -1097,15 +1097,6 @@ def test_make_context_factory_no_method_verify_none(self):
         self.assertEqual(
             self.get_context(context_factory).get_verify_mode(), VERIFY_NONE)
 
-    def test_make_context_factory_sslv3_verify_none(self):
-        context_factory = make_context_factory(
-            verify_options=VERIFY_NONE, ssl_method=SSLv3_METHOD)
-        self.assertIsInstance(context_factory, HttpClientContextFactory)
-        self.assertEqual(context_factory.verify_options, VERIFY_NONE)
-        self.assertEqual(context_factory.ssl_method, SSLv3_METHOD)
-        self.assertEqual(
-            self.get_context(context_factory).get_verify_mode(), VERIFY_NONE)
-
     def test_make_context_factory_no_method_verify_peer(self):
         # This test's behaviour depends on the version of Twisted being used.
         context_factory = make_context_factory(verify_options=VERIFY_PEER)
@@ -1150,17 +1141,6 @@ def test_make_context_factory_no_method_no_verify(self):
         else:
             self.assertIsInstance(context_factory, HttpClientPolicyForHTTPS)
 
-    def test_make_context_factory_sslv3_no_verify(self):
-        # This test's behaviour depends on the version of Twisted being used.
-        context_factory = make_context_factory(ssl_method=SSLv3_METHOD)
-        self.assertEqual(context_factory.ssl_method, SSLv3_METHOD)
-        if HttpClientPolicyForHTTPS is None:
-            # We have Twisted<14.0.0
-            self.assertIsInstance(context_factory, HttpClientContextFactory)
-            self.assertEqual(context_factory.verify_options, None)
-        else:
-            self.assertIsInstance(context_factory, HttpClientPolicyForHTTPS)
-
     @inlineCallbacks
     def test_handle_get(self):
         self.http_request_succeed("foo")
@@ -1337,18 +1317,6 @@ def test_https_request_method_default(self):
         context_factory = self.get_context_factory()
         self.assertEqual(context_factory.ssl_method, None)
 
-    @inlineCallbacks
-    def test_https_request_method_SSLv3(self):
-        self.http_request_succeed("foo")
-        reply = yield self.dispatch_command(
-            'get', url='https://www.example.com', ssl_method='SSLv3')
-        self.assertTrue(reply['success'])
-        self.assertEqual(reply['body'], "foo")
-        self.assert_http_request('https://www.example.com', method='GET')
-
-        context_factory = self.get_context_factory()
-        self.assertEqual(context_factory.ssl_method, SSLv3_METHOD)
-
     @inlineCallbacks
     def test_https_request_method_SSLv23(self):
         self.http_request_succeed("foo")

diff --git a/vumi/components/tests/test_tagpool_api.py b/vumi/components/tests/test_tagpool_api.py
@@ -128,7 +128,10 @@ def test_purge_pool(self):
     def test_purge_pool_with_keys_in_use(self):
         d = self.proxy.callRemote("purge_pool", "pool2")
         yield d.addErrback(lambda f: log.err(f))
-        errors = self.flushLoggedErrors('txjsonrpc.jsonrpclib.Fault')
+        # txJSON-RPC 0.5 adds support for py3 which means
+        # different error classes are being logged, accept both
+        errors = self.flushLoggedErrors('xmlrpclib.Fault',
+                                        'xmlrpc.client.Fault')
         self.assertEqual(len(errors), 1)
         server_errors = self.flushLoggedErrors(
             'vumi.components.tagpool.TagpoolError')

diff --git a/vumi/middleware/base.py b/vumi/middleware/base.py
@@ -171,6 +171,11 @@ def handle_failure(self, failure, connector_name):
         """
         return failure
 
+    def handle_publish_status(self, status, connector_name):
+        """Called when a status is published.
+        """
+        return status
+
 
 class TransportMiddleware(BaseMiddleware):
     """Message processor middleware for Transports.

diff --git a/vumi/tests/fake_connection.py b/vumi/tests/fake_connection.py
@@ -271,7 +271,7 @@ def __init__(self, handler):
     def endpoint(self):
         return self.fake_server.endpoint
 
-    def get_agent(self, reactor=None, contextFactory=None):
+    def get_agent(self, reactor=None, pool=None, contextFactory=None):
         """
         Returns an IAgent that makes requests to this fake server.
         """
@@ -297,6 +297,7 @@ def render_PUT(self, request):
 
 
 class ProxyAgentWithContext(ProxyAgent):
-    def __init__(self, endpoint, reactor=None, contextFactory=None):
+    def __init__(self, endpoint, reactor=None, pool=None, contextFactory=None):
         self.contextFactory = contextFactory  # To assert on in tests.
-        super(ProxyAgentWithContext, self).__init__(endpoint, reactor=reactor)
+        super(ProxyAgentWithContext, self).__init__(
+            endpoint, reactor=reactor, pool=pool)
diff --git a/vumi/tests/test_utils.py b/vumi/tests/test_utils.py
@@ -234,9 +234,9 @@ def test_http_request_with_custom_context_factory(self):
 
         ctxt = WebClientContextFactory()
 
-        def stashing_factory(reactor, contextFactory=None):
+        def stashing_factory(reactor, contextFactory=None, pool=None):
             agent = self.fake_http.get_agent(
-                reactor, contextFactory=contextFactory)
+                reactor, contextFactory=contextFactory, pool=pool)
             agents.append(agent)
             return agent
 

diff --git a/vumi/transports/dmark/dmark_ussd.py b/vumi/transports/dmark/dmark_ussd.py
@@ -6,7 +6,7 @@
 from twisted.web import http
 
 from vumi.components.session import SessionManager
-from vumi.config import ConfigDict, ConfigInt
+from vumi.config import ConfigDict, ConfigInt, ConfigBool
 from vumi.message import TransportUserMessage
 from vumi.transports.httprpc import HttpRpcTransport
 
@@ -19,6 +19,10 @@ class DmarkUssdTransportConfig(HttpRpcTransport.CONFIG_CLASS):
         " expires.",
         default=600, static=True)
 
+    fix_to_addr = ConfigBool(
+        "Whether or not to ensure that the to_addr is always starting with a "
+        "* and ending with a #", default=False, static=True)
+
     redis_manager = ConfigDict(
         "Redis client configuration.", default={}, static=True)
 
@@ -150,7 +154,12 @@ def handle_raw_inbound_message(self, request_id, request):
             type='request_parsed',
             message='Request parsed',)
 
+        config = self.get_static_config()
         to_addr = values["ussdServiceCode"]
+        if (config.fix_to_addr
+            and not to_addr.startswith('*')
+                and not to_addr.endswith('#')):
+            to_addr = '*%s#' % (to_addr,)
         from_addr = values["msisdn"]
         content = values["ussdRequestString"]
         session_event = yield self.session_event_for_transaction(
@@ -166,6 +175,9 @@ def handle_raw_inbound_message(self, request_id, request):
             provider='dmark',
             session_event=session_event,
             transport_type=self.transport_type,
+            helper_metadata={
+                'session_id': values['transactionId'],
+            },
             transport_metadata={
                 'dmark_ussd': {
                     'transaction_id': values['transactionId'],