Merge branch 'develop' into release/0.6.x

docker/Dockerfile

@@ -1,11 +1,10 @@
 FROM praekeltfoundation/python-base
 MAINTAINER Praekelt Foundation <dev@praekeltfoundation.org>
 
-ENV VUMI_VERSION "0.6.2"
+ENV VUMI_VERSION "0.6.9"
 RUN pip install vumi==$VUMI_VERSION
 
 COPY ./vumi-entrypoint.sh /app/vumi-entrypoint.sh
 WORKDIR /app
 
-ENTRYPOINT ["eval-args.sh", "dinit", "/app/vumi-entrypoint.sh"]
-CMD []
+CMD ["/app/vumi-entrypoint.sh"]

docker/vumi-entrypoint.sh

@@ -1,30 +1,21 @@
-#!/bin/bash -e
+#!/usr/bin/env bash
+set -e
 
 TWISTD_COMMAND="${TWISTD_COMMAND:-vumi_worker}"
 
-WORKER_CLASS_OPT=""
-if [ -n "$WORKER_CLASS" ]; then
-  WORKER_CLASS_OPT="--worker-class $WORKER_CLASS"
-fi
-
-CONFIG_OPT=""
-if [ -n "$CONFIG_FILE" ]; then
-  CONFIG_OPT="--config $CONFIG_FILE"
-fi
+WORKER_CLASS_OPT="${WORKER_CLASS:+--worker-class $WORKER_CLASS}"
+CONFIG_OPT="${CONFIG_FILE:+--config $CONFIG_FILE}"
 
 AMQP_OPTS=""
-if [ -n "$AMQP_HOST" ]; then
+if [[ -n "$AMQP_HOST" ]]; then
   AMQP_OPTS="--hostname $AMQP_HOST \
     --port ${AMQP_PORT:-5672} \
     --vhost ${AMQP_VHOST:-/} \
     --username ${AMQP_USERNAME:-guest} \
     --password ${AMQP_PASSWORD:-guest}"
 fi
 
-SENTRY_OPT=""
-if [ -n "$SENTRY_DSN" ]; then
-  SENTRY_OPT="--sentry $SENTRY_DSN"
-fi
+SENTRY_OPT="${SENTRY_DSN:+--sentry $SENTRY_DSN}"
 
 SET_OPTS=$(env | grep ^VUMI_OPT_ | sed -e 's/^VUMI_OPT_//' -e 's/=/ /' | awk '{printf("%s=%s:%s ", "--set-option", tolower($1), $2);}')
 

docs/conf.py

@@ -53,7 +53,7 @@
 # The short X.Y version.
 version = '0.6'
 # The full version, including alpha/beta/rc tags.
-release = '0.6.7'
+release = '0.6.9'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/release-notes.rst

@@ -11,6 +11,25 @@ this will almost certainly not break the majority of things built on vumi, old
 code or code that relies too heavily on the details of worker setup may need to
 be fixed.
 
+:Version: 0.6.9
+:Date released: 27 July 2016
+
+* Apply numerous cleanups to the Dockerfile.
+* Use only decimal digits for session identifiers in the MTN Nigeria USSD
+  XML over TCP transport.
+* Add the ability to configure the PDU field the dialed USSD code is taken
+  from in the 6D SMPP processor.
+* Update tests to pass with Twisted 16.3.
+
+:Version: 0.6.8
+:Date released: 12 May 2016
+
+* Allow disabling of delivery report handling as sometimes these cause more noise
+  than signal.
+* Embed the original SMPP transports delivery report status into the message
+  transport metadata. This is useful information that applications may chose
+  to act on.
+
 :Version: 0.6.7
 :Date released: 19 April 2016
 

setup.py

@@ -12,7 +12,7 @@
 
 setup(
     name="vumi",
-    version="0.6.7",
+    version="0.6.9",
     url='http://github.com/praekelt/vumi',
     license='BSD',
     description="Super-scalable messaging engine for the delivery of SMS, "

utils/bump-version.sh

@@ -22,3 +22,4 @@ inplace_sed "s/\(__version__[ ]*=[ ]*[\"']\)\(.*\)\([\"'].*\)/\1${VER}\3/" vumi/
 inplace_sed "s/\(version[ ]*=[ ]*[\"']\)\(.*\)\([\"'].*\)/\1${VER}\3/" setup.py
 inplace_sed "s/^\(release[ ]*=[ ]*[\"']\)\(.*\)\([\"'].*\)/\1${VER}\3/" docs/conf.py
 inplace_sed "s/^\(version[ ]*=[ ]*[\"']\)\(.*\)\([\"'].*\)/\1${SHORT_VER}\3/" docs/conf.py
+inplace_sed "s/^\(ENV [ ]*VUMI_VERSION [ ]*\"\)\([^\"]*\)\(\".*\)/\1${VER}\3/" docker/Dockerfile

vumi/__init__.py

@@ -2,4 +2,4 @@
 Vumi scalable text messaging engine.
 """
 
-__version__ = "0.6.7"
+__version__ = "0.6.9"

vumi/components/message_store_resource.py

@@ -155,7 +155,8 @@ def handle_message(self, message_key, request):
         return d
 
     def write_message(self, message, request):
-        self.formatter.write_row(request, message)
+        if not request.content.closed:
+            self.formatter.write_row(request, message)
 
 
 class InboundResource(MessageStoreProxyResource):

vumi/transports/mtn_nigeria/tests/test_xml_over_tcp.py

@@ -856,3 +856,9 @@ def test_error_response_handling_for_unknown_codes(self):
             'err',
             "Server sent error message: (1337) Unknown Code: "
             "Some Reason")
+
+    def test_gen_session_id(self):
+        sessid = XmlOverTcpClient.gen_session_id()
+        self.assertEqual(len(sessid), XmlOverTcpClient.SESSION_ID_HEADER_SIZE)
+        self.assertTrue(
+            all(c in XmlOverTcpClient.SESSION_ID_CHARACTERS for c in sessid))

vumi/transports/mtn_nigeria/xml_over_tcp.py

@@ -1,6 +1,5 @@
-import uuid
+import random
 import struct
-from random import randint
 
 from twisted.web import microdom
 from twisted.internet import reactor
@@ -58,6 +57,7 @@ class XmlOverTcpClient(Protocol):
     LENGTH_HEADER_SIZE = 16
     HEADER_SIZE = SESSION_ID_HEADER_SIZE + LENGTH_HEADER_SIZE
     HEADER_FORMAT = '!%ss%ss' % (SESSION_ID_HEADER_SIZE, LENGTH_HEADER_SIZE)
+    SESSION_ID_CHARACTERS = "0123456789"
 
     REQUEST_ID_LENGTH = 10
 
@@ -407,17 +407,16 @@ def gen_session_id(cls):
         """
         Generates session id. Used for packets needing a dummy session id.
         """
-        # NOTE: Slicing the generated uuid is probably a bad idea, and will
-        # affect collision resistence, but I can't think of a simpler way to
-        # generate a unique 16 char alphanumeric.
-        return uuid.uuid4().hex[:cls.SESSION_ID_HEADER_SIZE]
+        return "".join(
+            random.choice(cls.SESSION_ID_CHARACTERS)
+            for i in range(cls.SESSION_ID_HEADER_SIZE))
 
     @classmethod
     def gen_request_id(cls):
         # NOTE: The protocol requires request ids to be number only ids. With a
         # request id length of 10 digits, generating ids using randint could
         # well cause collisions to occur, although this should be unlikely.
-        return str(randint(0, (10 ** cls.REQUEST_ID_LENGTH) - 1))
+        return str(random.randint(0, (10 ** cls.REQUEST_ID_LENGTH) - 1))
 
     def login(self):
         params = [

vumi/transports/mxit/tests/test_mxit.py

@@ -2,17 +2,27 @@
 import base64
 
 from twisted.internet.defer import inlineCallbacks, DeferredQueue
-from twisted.web.http import Request, BAD_REQUEST
+from twisted.web.http import BAD_REQUEST
 from twisted.web.server import NOT_DONE_YET
+from twisted.web.http_headers import Headers
 
 from vumi.transports.mxit import MxitTransport
 from vumi.transports.mxit.responses import ResponseParser
 from vumi.utils import http_request_full
 from vumi.tests.helpers import VumiTestCase
 from vumi.tests.fake_connection import FakeHttpServer
+from twisted.web.test.requesthelper import DummyRequest as TwistedDummyRequest
 from vumi.transports.tests.helpers import TransportHelper
 
 
+class DummyRequest(TwistedDummyRequest):
+    def __init__(self, *args, **kw):
+        # Twisted 13.2.0 doesn't have .requestHeaders on DummyRequest
+        TwistedDummyRequest.__init__(self, *args, **kw)
+        if not hasattr(self, 'requestHeaders'):
+            self.requestHeaders = Headers()
+
+
 class TestMxitTransport(VumiTestCase):
 
     @inlineCallbacks
@@ -62,7 +72,7 @@ def handle_request(self, request):
         return NOT_DONE_YET
 
     def test_is_mxit_request(self):
-        req = Request(None, True)
+        req = DummyRequest([])
         self.assertFalse(self.transport.is_mxit_request(req))
         req.requestHeaders.addRawHeader('X-Mxit-Contact', 'foo')
         self.assertTrue(self.transport.is_mxit_request(req))
@@ -100,7 +110,7 @@ def test_html_decode(self):
             self.transport.html_decode(self.sample_html_str), '<&>')
 
     def test_get_request_data(self):
-        req = Request(None, True)
+        req = DummyRequest([])
         headers = req.requestHeaders
         for key, value in self.sample_req_headers.items():
             headers.addRawHeader(key, value)
@@ -134,29 +144,29 @@ def test_get_request_data(self):
         })
 
     def test_get_request_content_from_header(self):
-        req = Request(None, True)
+        req = DummyRequest([])
         req.requestHeaders.addRawHeader('X-Mxit-User-Input', 'foo')
         self.assertEqual(self.transport.get_request_content(req), 'foo')
 
     def test_get_quote_plus_request_content_from_header(self):
-        req = Request(None, True)
+        req = DummyRequest([])
         req.requestHeaders.addRawHeader('X-Mxit-User-Input', 'foo+bar')
         self.assertEqual(
             self.transport.get_request_content(req), 'foo bar')
 
     def test_get_quoted_request_content_from_header(self):
-        req = Request(None, True)
+        req = DummyRequest([])
         req.requestHeaders.addRawHeader('X-Mxit-User-Input', 'foo%20bar')
         self.assertEqual(
             self.transport.get_request_content(req), 'foo bar')
 
     def test_get_request_content_from_args(self):
-        req = Request(None, True)
+        req = DummyRequest([])
         req.args = {'input': ['bar']}
         self.assertEqual(self.transport.get_request_content(req), 'bar')
 
     def test_get_request_content_when_missing(self):
-        req = Request(None, True)
+        req = DummyRequest([])
         self.assertEqual(self.transport.get_request_content(req), None)
 
     @inlineCallbacks

vumi/transports/smpp/config.py

@@ -32,6 +32,13 @@ class SmppTransportConfig(Transport.CONFIG_CLASS):
         'causes more noise than signal. It can optionally be turned off. '
         'Defaults to False.',
         default=False, static=True)
+    disable_delivery_report = ConfigBool(
+        'Disable publishing of `delivery_report` events. In some cases this '
+        'event causes more noise than signal. It can optionally be turned '
+        'off. Note that failed or successful delivery reports will still be '
+        'used to track which SMPP message ids can be removed from temporary '
+        'caches. Defaults to False.',
+        default=False, static=True)
     third_party_id_expiry = ConfigInt(
         'How long (in seconds) to keep 3rd party message IDs around to allow '
         'for matching submit_sm_resp and delivery report messages. Defaults '

vumi/transports/smpp/processors/default.py

@@ -102,7 +102,8 @@ def _handle_delivery_report_optional_params(self, pdu):
 
         d = self.transport.handle_delivery_report(
             receipted_message_id=receipted_message_id,
-            delivery_status=self.delivery_status(status))
+            delivery_status=self.delivery_status(status),
+            smpp_delivery_status=status)
         d.addCallback(lambda _: True)
         return d
 
@@ -115,7 +116,8 @@ def _process_delivery_report_content_fields(self, content_fields):
         message_state = content_fields['stat']
         return self.transport.handle_delivery_report(
             receipted_message_id=receipted_message_id,
-            delivery_status=self.delivery_status(message_state))
+            delivery_status=self.delivery_status(message_state),
+            smpp_delivery_status=message_state)
 
     def _handle_delivery_report_esm_class(self, pdu):
         """

vumi/transports/smpp/processors/sixdee.py

@@ -1,6 +1,6 @@
 # -*- test-case-name: vumi.transports.smpp.tests.test_sixdee -*-
 
-from vumi.config import ConfigInt
+from vumi.config import ConfigInt, ConfigText
 from vumi.components.session import SessionManager
 from vumi.message import TransportUserMessage
 from vumi.transports.smpp.processors import default
@@ -19,6 +19,11 @@ class DeliverShortMessageProcessorConfig(
         'Maximum length a USSD sessions data is to be kept for in seconds.',
         default=60 * 3, static=True)
 
+    ussd_code_pdu_field = ConfigText(
+        'PDU field to read the message `to_addr` (USSD code) from. Possible'
+        ' options are "short_message" (the default) and "destination_addr".',
+        default='short_message', static=True)
+
 
 class DeliverShortMessageProcessor(default.DeliverShortMessageProcessor):
 
@@ -63,7 +68,7 @@ def handle_deliver_sm_ussd(self, pdu, pdu_params, pdu_opts):
 
         if session_event == 'new':
             # PSSR request. Let's assume it means a new session.
-            ussd_code = pdu_params['short_message']
+            ussd_code = pdu_params[self.config.ussd_code_pdu_field]
             content = None
 
             yield self.session_manager.create_session(

vumi/transports/smpp/processors/tests/test_sixdee.py

@@ -70,8 +70,10 @@ def setUp(self):
         }
 
     @inlineCallbacks
-    def get_transport(self, config={}, bind=True):
+    def get_transport(self, deliver_config={}, submit_config={}, bind=True):
         cfg = self.default_config.copy()
+        cfg['deliver_short_message_processor_config'].update(deliver_config)
+        cfg['submit_short_message_processor_config'].update(submit_config)
         transport = yield self.tx_helper.get_transport(cfg, start=False)
         transport.clock = self.clock
         yield transport.startWorker()
@@ -81,9 +83,15 @@ def get_transport(self, config={}, bind=True):
         returnValue(transport)
 
     def assert_udh_parts(self, pdus, texts, encoding):
-        pdu_header = lambda pdu: short_message(pdu)[:6]
-        pdu_text = lambda pdu: short_message(pdu)[6:].decode(encoding)
-        udh_header = lambda i: '\x05\x00\x03\x03\x07' + chr(i)
+        def pdu_header(pdu):
+            return short_message(pdu)[:6]
+
+        def pdu_text(pdu):
+            return short_message(pdu)[6:].decode(encoding)
+
+        def udh_header(i):
+            return '\x05\x00\x03\x03\x07' + chr(i)
+
         self.assertEqual(
             [(pdu_header(pdu), pdu_text(pdu)) for pdu in pdus],
             [(udh_header(i + 1), text) for i, text in enumerate(texts)])
@@ -149,6 +157,36 @@ def test_submit_and_deliver_ussd_new(self):
                 }
             })
 
+    @inlineCallbacks
+    def test_submit_and_deliver_ussd_new_custom_ussd_code_field(self):
+        session = SessionInfo()
+        yield self.get_transport(deliver_config={
+            'ussd_code_pdu_field': 'destination_addr',
+        })
+
+        # Server delivers a USSD message to the Client
+        pdu = DeliverSM(1, short_message="*IGNORE#", destination_addr="*123#")
+        pdu.add_optional_parameter('ussd_service_op', '01')
+        pdu.add_optional_parameter('its_session_info', session.its_info)
+
+        yield self.fake_smsc.handle_pdu(pdu)
+
+        [mess] = yield self.tx_helper.wait_for_dispatched_inbound(1)
+
+        self.assertEqual(mess['content'], None)
+        self.assertEqual(mess['to_addr'], '*123#')
+        self.assertEqual(mess['transport_type'], "ussd")
+        self.assertEqual(mess['session_event'],
+                         TransportUserMessage.SESSION_NEW)
+        self.assertEqual(
+            mess['transport_metadata'],
+            {
+                'session_info': {
+                    'session_identifier': session.sixdee_id,
+                    'ussd_service_op': '01',
+                }
+            })
+
     @inlineCallbacks
     def test_deliver_sm_op_codes_new(self):
         session = SessionInfo()