Merge branch 'develop' into feature/SEED-927-vumi-http-request-full-f…

…ails-sni

README.rst

@@ -10,9 +10,9 @@ be fixed.
 
 Documentation available online at http://vumi.readthedocs.org/ and in the `docs` directory of the repository.
 
-|vumi-ver| |vumi-ci| |vumi-cover| |python-ver| |vumi-docs| |vumi-downloads| |vumi-license|
+|vumi-ver| |vumi-ci| |vumi-cover| |python-ver| |vumi-docs| |vumi-license|
 
-.. |vumi-ver| image:: https://pypip.in/v/vumi/badge.png?text=pypi
+.. |vumi-ver| image:: https://img.shields.io/pypi/v/vumi.svg
     :alt: Vumi version
     :scale: 100%
     :target: https://pypi.python.org/pypi/vumi
@@ -27,7 +27,7 @@ Documentation available online at http://vumi.readthedocs.org/ and in the `docs`
     :scale: 100%
     :target: https://coveralls.io/r/praekelt/vumi
 
-.. |python-ver| image:: https://pypip.in/py_versions/vumi/badge.svg
+.. |python-ver| image:: https://img.shields.io/pypi/pyversions/vumi.svg
     :alt: Python version
     :scale: 100%
     :target: https://pypi.python.org/pypi/vumi
@@ -37,12 +37,13 @@ Documentation available online at http://vumi.readthedocs.org/ and in the `docs`
     :scale: 100%
     :target: http://vumi.readthedocs.org/
 
-.. |vumi-downloads| image:: https://pypip.in/download/vumi/badge.svg
+.. This is commented out because downloads stats from PyPI are not working
+.. .. |vumi-downloads| image:: https://img.shields.io/pypi/dm/vumi.svg
     :alt: Vumi downloads from PyPI
     :scale: 100%
     :target: https://pypi.python.org/pypi/vumi
 
-.. |vumi-license| image:: https://pypip.in/license/vumi/badge.svg
+.. |vumi-license| image:: https://img.shields.io/pypi/l/vumi.svg
     :target: https://pypi.python.org/pypi/vumi
     :alt: Vumi license
 

docker/Dockerfile

@@ -1,7 +1,7 @@
 FROM praekeltfoundation/python-base
 MAINTAINER Praekelt Foundation <dev@praekeltfoundation.org>
 
-ENV VUMI_VERSION "0.6.12"
+ENV VUMI_VERSION "0.6.14"
 RUN pip install vumi==$VUMI_VERSION
 
 COPY ./vumi-entrypoint.sh /app/vumi-entrypoint.sh

docs/conf.py

@@ -53,7 +53,7 @@
 # The short X.Y version.
 version = '0.6'
 # The full version, including alpha/beta/rc tags.
-release = '0.6.12'
+release = '0.6.14'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/release-notes.rst

@@ -11,6 +11,26 @@ this will almost certainly not break the majority of things built on vumi, old
 code or code that relies too heavily on the details of worker setup may need to
 be fixed.
 
+:Version: 0.6.13
+:Date released: 10 January 2017
+
+* Ensure that the keys for the data coding mapping of the SMPP transport get
+  converted to ints, as it's not possible with Junebug's JSON channel config to
+  represent dictionary keys as integers.
+* Update Message Sender tests to work with the new Riak client.
+
+:Version: 0.6.12
+:Date released: 23 September 2016
+
+* Update the HttpRPCTransport to use the new vumi logging to be compatible with
+  Junebug log collection.
+
+:Version: 0.6.11
+:Date released: 12 August 2016
+
+* Change logging level from warning to info for SMPP disconnections, to reduce
+  sentry entries when being throttled.
+
 :Version: 0.6.10
 :Date released: 27 July 2016
 

setup.py

@@ -12,7 +12,7 @@
 
 setup(
     name="vumi",
-    version="0.6.12",
+    version="0.6.14",
     url='http://github.com/praekelt/vumi',
     license='BSD',
     description="Super-scalable messaging engine for the delivery of SMS, "
@@ -52,7 +52,7 @@
         'riak>=2.1',
         'txJSON-RPC==0.3.1',
         'txTwitter>=0.1.4a',
-        'treq',
+        'treq<16.12.0',
         'confmodel>=0.2.0',
         'hyperloglog',
     ],

vumi/__init__.py

@@ -2,4 +2,4 @@
 Vumi scalable text messaging engine.
 """
 
-__version__ = "0.6.12"
+__version__ = "0.6.14"

vumi/persist/tests/test_riak_manager.py

@@ -15,13 +15,11 @@ class TestRiakManager(CommonRiakManagerTests, VumiTestCase):
 
     def setUp(self):
         try:
-            from vumi.persist.riak_manager import (
-                RiakManager, flatten_generator)
+            from vumi.persist.riak_manager import flatten_generator
         except ImportError, e:
             import_skip(e, 'riak')
         self.call_decorator = flatten_generator
-        self.manager = RiakManager.from_config({'bucket_prefix': 'test.'})
-        self.add_cleanup(self.manager.purge_all)
+        self.manager = self.create_riak_manager()
         self.manager.purge_all()
 
     def test_call_decorator(self):

vumi/persist/tests/test_txriak_manager.py

@@ -53,6 +53,45 @@ def mkdummy(self, key, data=None, dummy_class=DummyModel):
             dummy.set_data(data)
         return dummy
 
+    @inlineCallbacks
+    def purge_txriak(self):
+        """
+        Creates a manager, purges all the riak data with it, then closes it.
+        """
+        manager = self.create_txriak_manager()
+        yield manager.purge_all()
+        yield manager.close_manager()
+
+    def create_txriak_manager(self):
+        """
+        Creates and returns a TxRiakManager, handling cleanup.
+        """
+        try:
+            from vumi.persist.txriak_manager import TxRiakManager
+        except ImportError, e:
+            import_skip(e, 'riak', 'riak')
+        self.add_cleanup(self.purge_txriak)
+        return TxRiakManager.from_config({'bucket_prefix': 'test.'})
+
+    def purge_riak(self):
+        """
+        Creates a manager, purges all the riak data with it, then closes it.
+        """
+        manager = self.create_riak_manager()
+        manager.purge_all()
+        manager.close_manager()
+
+    def create_riak_manager(self):
+        """
+        Creates and returns a RiakManager, handling cleanup.
+        """
+        try:
+            from vumi.persist.riak_manager import RiakManager
+        except ImportError, e:
+            import_skip(e, 'riak', 'riak')
+        self.add_cleanup(self.purge_riak)
+        return RiakManager.from_config({'bucket_prefix': 'test.'})
+
     def test_from_config(self):
         manager_cls = self.manager.__class__
         manager = manager_cls.from_config({'bucket_prefix': 'test.'})
@@ -338,12 +377,7 @@ class TestTxRiakManager(CommonRiakManagerTests, VumiTestCase):
 
     @inlineCallbacks
     def setUp(self):
-        try:
-            from vumi.persist.txriak_manager import TxRiakManager
-        except ImportError, e:
-            import_skip(e, 'riak', 'riak')
-        self.manager = TxRiakManager.from_config({'bucket_prefix': 'test.'})
-        self.add_cleanup(self.manager.purge_all)
+        self.manager = self.create_txriak_manager()
         yield self.manager.purge_all()
 
     def test_call_decorator(self):

vumi/tests/helpers.py

@@ -1454,7 +1454,12 @@ def cleanup(self):
                 unclosed_managers.append(manager)
             if purge:
                 try:
-                    yield self._purge_riak(manager)
+                    # Create a new client for cleanup
+                    purge_manager = self.get_riak_manager(config={
+                        'bucket_prefix': manager.bucket_prefix
+                    })
+                    yield self._purge_riak(purge_manager)
+                    yield purge_manager.close_manager()
                 except ConnectionRefusedError:
                     pass
             yield manager.close_manager()

vumi/tests/test_test_helpers.py

@@ -2180,38 +2180,47 @@ def test__purge_redis_not_connected(self):
         success_result_of(persistence_helper._purge_redis(manager))
         self.assertEqual(manager.purged, False)
 
+    @inlineCallbacks
     def test_cleanup_purges_managers(self):
         """
         .cleanup() should purge the Riak and Redis managers that need purging.
         """
-        persistence_helper = PersistenceHelper()
-        riak_purge = FakeRiakManagerForCleanup('bucket1')
-        riak_nopurge = FakeRiakManagerForCleanup('bucket1')
+        riak_purged = []
+
+        persistence_helper = PersistenceHelper(use_riak=True)
+        persistence_helper.setup()
+        persistence_helper._purge_riak = riak_purged.append
+
+        yield persistence_helper.get_riak_manager(
+            config={'bucket_prefix': 'bucket1'}).close_manager()
+        yield persistence_helper.get_riak_manager(
+            config={'bucket_prefix': 'bucket1'}).close_manager()
+
         redis_purge = FakeRedisManagerForCleanup('prefix1')
         redis_nopurge = FakeRedisManagerForCleanup('prefix1')
-        persistence_helper._riak_managers.extend([riak_purge, riak_nopurge])
         persistence_helper._redis_managers.extend([redis_purge, redis_nopurge])
 
-        success_result_of(persistence_helper.cleanup())
-        self.assertEqual(
-            [True, False], [riak_purge.purged, riak_nopurge.purged])
+        yield persistence_helper.cleanup()
+        self.assertEqual(len(riak_purged), 1)
         self.assertEqual(
             [True, False], [redis_purge.purged, redis_nopurge.purged])
 
+    @inlineCallbacks
     def test_cleanup_closes_riak_managers(self):
         """
         .cleanup() should close Riak client connections.
         """
-        persistence_helper = PersistenceHelper()
-        conn1 = FakeRiakClientConnection()
-        conn2 = FakeRiakClientConnection()
-        manager = FakeRiakManagerForCleanup('bucket1', [conn1, conn2])
-        persistence_helper._riak_managers.append(manager)
-        self.assertEqual(manager.fake_conns, [conn1, conn2])
-        self.assertEqual([conn1.closed, conn2.closed], [False, False])
-        success_result_of(persistence_helper.cleanup())
-        self.assertEqual(manager.fake_conns, [])
-        self.assertEqual([conn1.closed, conn2.closed], [True, True])
+        persistence_helper = PersistenceHelper(use_riak=True)
+        persistence_helper.setup()
+        # Manually override assert_closed because we want to check whether the
+        # cleanup function closes, no matter what the env var is set to.
+        persistence_helper._assert_closed = False
+
+        manager = persistence_helper.get_riak_manager()
+        self.assertEqual(manager.client._closed, False)
+
+        yield persistence_helper.cleanup()
+        self.assertEqual(manager.client._closed, True)
 
     def test_record_load_and_store(self):
         """

vumi/transports/dmark/dmark_ussd.py

@@ -6,7 +6,7 @@
 from twisted.web import http
 
 from vumi.components.session import SessionManager
-from vumi.config import ConfigDict, ConfigInt
+from vumi.config import ConfigDict, ConfigInt, ConfigBool
 from vumi.message import TransportUserMessage
 from vumi.transports.httprpc import HttpRpcTransport
 
@@ -19,6 +19,10 @@ class DmarkUssdTransportConfig(HttpRpcTransport.CONFIG_CLASS):
         " expires.",
         default=600, static=True)
 
+    fix_to_addr = ConfigBool(
+        "Whether or not to ensure that the to_addr is always starting with a "
+        "* and ending with a #", default=False, static=True)
+
     redis_manager = ConfigDict(
         "Redis client configuration.", default={}, static=True)
 
@@ -150,7 +154,12 @@ def handle_raw_inbound_message(self, request_id, request):
             type='request_parsed',
             message='Request parsed',)
 
+        config = self.get_static_config()
         to_addr = values["ussdServiceCode"]
+        if (config.fix_to_addr
+            and not to_addr.startswith('*')
+                and not to_addr.endswith('#')):
+            to_addr = '*%s#' % (to_addr,)
         from_addr = values["msisdn"]
         content = values["ussdRequestString"]
         session_event = yield self.session_event_for_transaction(