From 14343e07785866f54721e2fa50a4832bf4bdd3af Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Tue, 19 Apr 2016 14:56:56 +0200 Subject: [PATCH 01/25] Update default URL to nostream API. --- vumi/transports/vumi_bridge/vumi_bridge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vumi/transports/vumi_bridge/vumi_bridge.py b/vumi/transports/vumi_bridge/vumi_bridge.py index 91e0ba383..36a290d7c 100644 --- a/vumi/transports/vumi_bridge/vumi_bridge.py +++ b/vumi/transports/vumi_bridge/vumi_bridge.py @@ -33,7 +33,7 @@ class VumiBridgeTransportConfig(Transport.CONFIG_CLASS): required=True) base_url = ConfigText( 'The base URL for the API', static=True, - default='https://go.vumi.org/api/v1/go/http_api/') + default='https://go.vumi.org/api/v1/go/http_api_nostream/') message_life_time = ConfigInt( 'How long to keep message_ids around for.', static=True, default=48 * 60 * 60) # default is 48 hours. From ad9ce7f2be0898a21780afb8fbf9ccd7c359151c Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Thu, 12 May 2016 17:51:18 +0200 Subject: [PATCH 02/25] Add smpp delivery status to delivery report events. --- vumi/transports/smpp/processors/default.py | 6 ++++-- vumi/transports/smpp/smpp_transport.py | 9 +++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/vumi/transports/smpp/processors/default.py b/vumi/transports/smpp/processors/default.py index 037f58435..b04768fca 100644 --- a/vumi/transports/smpp/processors/default.py +++ b/vumi/transports/smpp/processors/default.py @@ -102,7 +102,8 @@ def _handle_delivery_report_optional_params(self, pdu): d = self.transport.handle_delivery_report( receipted_message_id=receipted_message_id, - delivery_status=self.delivery_status(status)) + delivery_status=self.delivery_status(status), + smpp_delivery_status=status) d.addCallback(lambda _: True) return d @@ -115,7 +116,8 @@ def _process_delivery_report_content_fields(self, content_fields): message_state = content_fields['stat'] return self.transport.handle_delivery_report( receipted_message_id=receipted_message_id, - delivery_status=self.delivery_status(message_state)) + delivery_status=self.delivery_status(message_state), + smpp_delivery_status=message_state) def _handle_delivery_report_esm_class(self, pdu): """ diff --git a/vumi/transports/smpp/smpp_transport.py b/vumi/transports/smpp/smpp_transport.py index 757d95910..2f503f9da 100644 --- a/vumi/transports/smpp/smpp_transport.py +++ b/vumi/transports/smpp/smpp_transport.py @@ -492,7 +492,9 @@ def handle_raw_inbound_message(self, **kwargs): return self.publish_message(**message).addErrback(self.log.err) @inlineCallbacks - def handle_delivery_report(self, receipted_message_id, delivery_status): + def handle_delivery_report( + self, receipted_message_id, delivery_status, + smpp_delivery_status): message_id = yield self.message_stash.get_internal_message_id( receipted_message_id) if message_id is None: @@ -504,7 +506,10 @@ def handle_delivery_report(self, receipted_message_id, delivery_status): dr = yield self.publish_delivery_report( user_message_id=message_id, - delivery_status=delivery_status) + delivery_status=delivery_status, + transport_metadata={ + 'smpp_delivery_status': smpp_delivery_status, + }) if delivery_status in ('delivered', 'failed'): yield self.message_stash.expire_remote_message_id( From 4e1d2821b77bff594737c03722909d3d45a46ce2 Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Thu, 12 May 2016 17:59:02 +0200 Subject: [PATCH 03/25] Add option for disabling delivery reports. --- vumi/transports/smpp/config.py | 7 +++++++ vumi/transports/smpp/smpp_transport.py | 16 ++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/vumi/transports/smpp/config.py b/vumi/transports/smpp/config.py index f608c8c73..29cb5de6a 100644 --- a/vumi/transports/smpp/config.py +++ b/vumi/transports/smpp/config.py @@ -32,6 +32,13 @@ class SmppTransportConfig(Transport.CONFIG_CLASS): 'causes more noise than signal. It can optionally be turned off. ' 'Defaults to False.', default=False, static=True) + disable_delivery_report = ConfigBool( + 'Disable publishing of `delivery_report` events. In some cases this ' + 'event causes more noise than signal. It can optionally be turned ' + 'off. Note that failed or successful delivery reports will still be ' + 'used to track which SMPP message ids can be removed from temporary ' + 'caches. Defaults to False.', + default=False, static=True) third_party_id_expiry = ConfigInt( 'How long (in seconds) to keep 3rd party message IDs around to allow ' 'for matching submit_sm_resp and delivery report messages. Defaults ' diff --git a/vumi/transports/smpp/smpp_transport.py b/vumi/transports/smpp/smpp_transport.py index 2f503f9da..30f8ab820 100644 --- a/vumi/transports/smpp/smpp_transport.py +++ b/vumi/transports/smpp/smpp_transport.py @@ -278,6 +278,7 @@ def setup_transport(self): self.submit_sm_processor = config.submit_short_message_processor( self, config.submit_short_message_processor_config) self.disable_ack = config.disable_ack + self.disable_delivery_report = config.disable_delivery_report self.message_stash = SmppMessageDataStash(self.redis, config) self.service = self.start_service() @@ -504,12 +505,15 @@ def handle_delivery_report( % self.transport_name) return - dr = yield self.publish_delivery_report( - user_message_id=message_id, - delivery_status=delivery_status, - transport_metadata={ - 'smpp_delivery_status': smpp_delivery_status, - }) + if self.disable_delivery_report: + dr = None + else: + dr = yield self.publish_delivery_report( + user_message_id=message_id, + delivery_status=delivery_status, + transport_metadata={ + 'smpp_delivery_status': smpp_delivery_status, + }) if delivery_status in ('delivered', 'failed'): yield self.message_stash.expire_remote_message_id( From da600e9b76631641afc8b1dd49d9ea97b9861ae5 Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Thu, 12 May 2016 19:29:17 +0200 Subject: [PATCH 04/25] Add test for disabling delivery report. --- .../smpp/tests/test_smpp_transport.py | 53 +++++++++++++++---- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/vumi/transports/smpp/tests/test_smpp_transport.py b/vumi/transports/smpp/tests/test_smpp_transport.py index 036b92b25..d957cc94d 100644 --- a/vumi/transports/smpp/tests/test_smpp_transport.py +++ b/vumi/transports/smpp/tests/test_smpp_transport.py @@ -663,14 +663,7 @@ def test_mt_sms_ack(self): self.assertEqual(event['sent_message_id'], 'foo') @inlineCallbacks - def test_mt_sms_disabled_ack(self): - yield self.get_transport({'disable_ack': True}) - msg = self.tx_helper.make_outbound('hello world') - yield self.tx_helper.dispatch_outbound(msg) - submit_sm_pdu = yield self.fake_smsc.await_pdu() - self.fake_smsc.send_pdu( - SubmitSMResp(sequence_number=seq_no(submit_sm_pdu), - message_id='foo')) + def assert_no_events(self): # NOTE: We can't test for the absence of an event in isolation but we # can test that for the presence of a second event only. fail_msg = self.tx_helper.make_outbound('hello fail') @@ -678,10 +671,22 @@ def test_mt_sms_disabled_ack(self): submit_sm_fail_pdu = yield self.fake_smsc.await_pdu() self.fake_smsc.send_pdu( SubmitSMResp(sequence_number=seq_no(submit_sm_fail_pdu), - message_id='foo', command_status='ESME_RINVDSTADR')) + message_id='__assert_no_events__', + command_status='ESME_RINVDSTADR')) [fail] = yield self.tx_helper.wait_for_dispatched_events(1) self.assertEqual(fail['event_type'], 'nack') + @inlineCallbacks + def test_mt_sms_disabled_ack(self): + yield self.get_transport({'disable_ack': True}) + msg = self.tx_helper.make_outbound('hello world') + yield self.tx_helper.dispatch_outbound(msg) + submit_sm_pdu = yield self.fake_smsc.await_pdu() + self.fake_smsc.send_pdu( + SubmitSMResp(sequence_number=seq_no(submit_sm_pdu), + message_id='foo')) + yield self.assert_no_events() + @inlineCallbacks def test_mt_sms_nack(self): yield self.get_transport() @@ -1633,6 +1638,36 @@ def test_delivery_report_pending_keep_stored_remote_id(self): "remote_id_ttl (%s) <= final_dr_third_party_id_expiry (23)" % (remote_id_ttl,)) + @inlineCallbacks + def test_disable_delivery_report_delivered_delete_stored_remote_id(self): + transport = yield self.get_transport({ + 'final_dr_third_party_id_expiry': 23, + 'disable_delivery_report': True, + }) + + yield transport.message_stash.set_remote_message_id('bar', 'foo') + remote_id_ttl = yield transport.redis.ttl(remote_message_key('foo')) + + self.assertTrue( + remote_id_ttl > 23, + "remote_id_ttl (%s) <= final_dr_third_party_id_expiry (23)" + % (remote_id_ttl,)) + + pdu = DeliverSM(sequence_number=1, esm_class=4) + pdu.add_optional_parameter('receipted_message_id', 'foo') + pdu.add_optional_parameter('message_state', 2) + yield self.fake_smsc.handle_pdu(pdu) + yield self.fake_smsc.await_pdu() + + yield self.assert_no_events() + + remote_id_ttl = yield transport.redis.ttl(remote_message_key('foo')) + + self.assertTrue( + remote_id_ttl <= 23, + "remote_id_ttl (%s) > final_dr_third_party_id_expiry (23)" + % (remote_id_ttl,)) + @inlineCallbacks def test_reconnect(self): transport = yield self.get_transport(bind=False) From 223117626d814182d5e0b3a684ecaff60461c0fd Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Thu, 12 May 2016 19:33:47 +0200 Subject: [PATCH 05/25] Add tests for sending of raw delivery status. --- .../smpp/tests/test_smpp_transport.py | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/vumi/transports/smpp/tests/test_smpp_transport.py b/vumi/transports/smpp/tests/test_smpp_transport.py index d957cc94d..cb89d5fb9 100644 --- a/vumi/transports/smpp/tests/test_smpp_transport.py +++ b/vumi/transports/smpp/tests/test_smpp_transport.py @@ -1571,7 +1571,7 @@ def test_delivery_report_delivered_delete_stored_remote_id(self): pdu.add_optional_parameter('message_state', 2) yield self.fake_smsc.handle_pdu(pdu) - yield self.tx_helper.wait_for_dispatched_events(1) + [dr] = yield self.tx_helper.wait_for_dispatched_events(1) remote_id_ttl = yield transport.redis.ttl(remote_message_key('foo')) @@ -1580,6 +1580,12 @@ def test_delivery_report_delivered_delete_stored_remote_id(self): "remote_id_ttl (%s) > final_dr_third_party_id_expiry (23)" % (remote_id_ttl,)) + self.assertEqual(dr['event_type'], u'delivery_report') + self.assertEqual(dr['delivery_status'], u'delivered') + self.assertEqual(dr['transport_metadata'], { + u'smpp_delivery_status': u'DELIVERED', + }) + @inlineCallbacks def test_delivery_report_failed_delete_stored_remote_id(self): transport = yield self.get_transport({ @@ -1600,7 +1606,7 @@ def test_delivery_report_failed_delete_stored_remote_id(self): pdu.add_optional_parameter('message_state', 8) yield self.fake_smsc.handle_pdu(pdu) - yield self.tx_helper.wait_for_dispatched_events(1) + [dr] = yield self.tx_helper.wait_for_dispatched_events(1) remote_id_ttl = yield transport.redis.ttl(remote_message_key('foo')) @@ -1609,6 +1615,12 @@ def test_delivery_report_failed_delete_stored_remote_id(self): "remote_id_ttl (%s) > final_dr_third_party_id_expiry (23)" % (remote_id_ttl,)) + self.assertEqual(dr['event_type'], u'delivery_report') + self.assertEqual(dr['delivery_status'], u'failed') + self.assertEqual(dr['transport_metadata'], { + u'smpp_delivery_status': u'REJECTED', + }) + @inlineCallbacks def test_delivery_report_pending_keep_stored_remote_id(self): transport = yield self.get_transport({ @@ -1629,7 +1641,7 @@ def test_delivery_report_pending_keep_stored_remote_id(self): pdu.add_optional_parameter('message_state', 1) yield self.fake_smsc.handle_pdu(pdu) - yield self.tx_helper.wait_for_dispatched_events(1) + [dr] = yield self.tx_helper.wait_for_dispatched_events(1) remote_id_ttl = yield transport.redis.ttl(remote_message_key('foo')) @@ -1638,6 +1650,12 @@ def test_delivery_report_pending_keep_stored_remote_id(self): "remote_id_ttl (%s) <= final_dr_third_party_id_expiry (23)" % (remote_id_ttl,)) + self.assertEqual(dr['event_type'], u'delivery_report') + self.assertEqual(dr['delivery_status'], u'pending') + self.assertEqual(dr['transport_metadata'], { + u'smpp_delivery_status': u'ENROUTE', + }) + @inlineCallbacks def test_disable_delivery_report_delivered_delete_stored_remote_id(self): transport = yield self.get_transport({ From 82e2b01cd6fdfa5a07a10e5d13a83a6d9b2cc532 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Thu, 12 May 2016 18:54:58 +0100 Subject: [PATCH 06/25] Version 0.6.8 --- docs/conf.py | 2 +- docs/release-notes.rst | 9 +++++++++ setup.py | 2 +- vumi/__init__.py | 2 +- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index e2a1c2755..b229a2c65 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -53,7 +53,7 @@ # The short X.Y version. version = '0.6' # The full version, including alpha/beta/rc tags. -release = '0.6.7' +release = '0.6.8' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/release-notes.rst b/docs/release-notes.rst index 25094dd98..f421c4009 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -11,6 +11,15 @@ this will almost certainly not break the majority of things built on vumi, old code or code that relies too heavily on the details of worker setup may need to be fixed. +:Version: 0.6.8 +:Date released: 12 May 2016 + +* Allow disabling of delivery report handling as sometimes these cause more noise + than signal. +* Embed the original SMPP transports delivery report status into the message + transport metadata. This is useful information that applications may chose + to act on. + :Version: 0.6.7 :Date released: 19 April 2016 diff --git a/setup.py b/setup.py index 8722253cd..d3f00101c 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name="vumi", - version="0.6.7", + version="0.6.8", url='http://github.com/praekelt/vumi', license='BSD', description="Super-scalable messaging engine for the delivery of SMS, " diff --git a/vumi/__init__.py b/vumi/__init__.py index f81f7bb8b..c8bdce6ea 100644 --- a/vumi/__init__.py +++ b/vumi/__init__.py @@ -2,4 +2,4 @@ Vumi scalable text messaging engine. """ -__version__ = "0.6.7" +__version__ = "0.6.8" From 22339cbeff5b4f3796abd8227fff8e646d9df64f Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 14 Jun 2016 11:20:34 +0200 Subject: [PATCH 07/25] docker: Bump Vumi version --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 520c3a497..75fa73bc5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ FROM praekeltfoundation/python-base MAINTAINER Praekelt Foundation -ENV VUMI_VERSION "0.6.2" +ENV VUMI_VERSION "0.6.8" RUN pip install vumi==$VUMI_VERSION COPY ./vumi-entrypoint.sh /app/vumi-entrypoint.sh From 32a4b7de4d4c6a8d7bad79d49f1f9c6e7ce1c1fc Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 14 Jun 2016 11:21:12 +0200 Subject: [PATCH 08/25] docker: Get bash from env --- docker/vumi-entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/vumi-entrypoint.sh b/docker/vumi-entrypoint.sh index e18f4fb2d..e65cb3ffa 100755 --- a/docker/vumi-entrypoint.sh +++ b/docker/vumi-entrypoint.sh @@ -1,4 +1,5 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e TWISTD_COMMAND="${TWISTD_COMMAND:-vumi_worker}" From 7979a1ce679e2fca3839cc79971837e13104efce Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 14 Jun 2016 11:23:06 +0200 Subject: [PATCH 09/25] docker: Use shell string expansion rather than ifs --- docker/vumi-entrypoint.sh | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/docker/vumi-entrypoint.sh b/docker/vumi-entrypoint.sh index e65cb3ffa..22ce2f716 100755 --- a/docker/vumi-entrypoint.sh +++ b/docker/vumi-entrypoint.sh @@ -3,15 +3,8 @@ set -e TWISTD_COMMAND="${TWISTD_COMMAND:-vumi_worker}" -WORKER_CLASS_OPT="" -if [ -n "$WORKER_CLASS" ]; then - WORKER_CLASS_OPT="--worker-class $WORKER_CLASS" -fi - -CONFIG_OPT="" -if [ -n "$CONFIG_FILE" ]; then - CONFIG_OPT="--config $CONFIG_FILE" -fi +WORKER_CLASS_OPT="${WORKER_CLASS:+--worker-class $WORKER_CLASS}" +CONFIG_OPT="${CONFIG_FILE:+--config $CONFIG_FILE}" AMQP_OPTS="" if [ -n "$AMQP_HOST" ]; then @@ -22,10 +15,7 @@ if [ -n "$AMQP_HOST" ]; then --password ${AMQP_PASSWORD:-guest}" fi -SENTRY_OPT="" -if [ -n "$SENTRY_DSN" ]; then - SENTRY_OPT="--sentry $SENTRY_DSN" -fi +SENTRY_OPT="${SENTRY_DSN:+--sentry $SENTRY_DSN}" SET_OPTS=$(env | grep ^VUMI_OPT_ | sed -e 's/^VUMI_OPT_//' -e 's/=/ /' | awk '{printf("%s=%s:%s ", "--set-option", tolower($1), $2);}') From 4282cff4988547b7ac5a5ff41d79e3e72c59a1d5 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 14 Jun 2016 11:27:05 +0200 Subject: [PATCH 10/25] docker: Use safer bash test --- docker/vumi-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/vumi-entrypoint.sh b/docker/vumi-entrypoint.sh index 22ce2f716..fb601354e 100755 --- a/docker/vumi-entrypoint.sh +++ b/docker/vumi-entrypoint.sh @@ -7,7 +7,7 @@ WORKER_CLASS_OPT="${WORKER_CLASS:+--worker-class $WORKER_CLASS}" CONFIG_OPT="${CONFIG_FILE:+--config $CONFIG_FILE}" AMQP_OPTS="" -if [ -n "$AMQP_HOST" ]; then +if [[ -n "$AMQP_HOST" ]]; then AMQP_OPTS="--hostname $AMQP_HOST \ --port ${AMQP_PORT:-5672} \ --vhost ${AMQP_VHOST:-/} \ From 66c7a5f8604b13947bf348a49f2a6accb35d17d6 Mon Sep 17 00:00:00 2001 From: Jamie Hewland Date: Tue, 14 Jun 2016 12:15:18 +0200 Subject: [PATCH 11/25] docker: Make entrypoint script the CMD --- docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 75fa73bc5..db7b198d1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -7,5 +7,4 @@ RUN pip install vumi==$VUMI_VERSION COPY ./vumi-entrypoint.sh /app/vumi-entrypoint.sh WORKDIR /app -ENTRYPOINT ["eval-args.sh", "dinit", "/app/vumi-entrypoint.sh"] -CMD [] +CMD ["/app/vumi-entrypoint.sh"] From 2f19db9817858f13e0be81812790c50cdaeeb5fd Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Mon, 25 Jul 2016 17:23:58 +0200 Subject: [PATCH 12/25] Add support for reading the USSD code from different fields. --- vumi/transports/smpp/processors/sixdee.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/vumi/transports/smpp/processors/sixdee.py b/vumi/transports/smpp/processors/sixdee.py index 7b11fccab..28f7b5767 100644 --- a/vumi/transports/smpp/processors/sixdee.py +++ b/vumi/transports/smpp/processors/sixdee.py @@ -1,6 +1,6 @@ # -*- test-case-name: vumi.transports.smpp.tests.test_sixdee -*- -from vumi.config import ConfigInt +from vumi.config import ConfigInt, ConfigText from vumi.components.session import SessionManager from vumi.message import TransportUserMessage from vumi.transports.smpp.processors import default @@ -19,6 +19,11 @@ class DeliverShortMessageProcessorConfig( 'Maximum length a USSD sessions data is to be kept for in seconds.', default=60 * 3, static=True) + ussd_code_pdu_field = ConfigText( + 'PDU field to read the message `to_addr` (USSD code) from. Possible' + ' options are "short_message" (the default) and "destination_addr".', + default='short_message', static=True) + class DeliverShortMessageProcessor(default.DeliverShortMessageProcessor): @@ -63,7 +68,7 @@ def handle_deliver_sm_ussd(self, pdu, pdu_params, pdu_opts): if session_event == 'new': # PSSR request. Let's assume it means a new session. - ussd_code = pdu_params['short_message'] + ussd_code = pdu_params[self.config.ussd_code_pdu_field] content = None yield self.session_manager.create_session( From 6610f13975a902d1e21d83300fcf103c7d2a2e62 Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Mon, 25 Jul 2016 17:25:19 +0200 Subject: [PATCH 13/25] Add test for changing the USSD code PDU field. --- .../smpp/processors/tests/test_sixdee.py | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/vumi/transports/smpp/processors/tests/test_sixdee.py b/vumi/transports/smpp/processors/tests/test_sixdee.py index 160f4335b..ac03d82a6 100644 --- a/vumi/transports/smpp/processors/tests/test_sixdee.py +++ b/vumi/transports/smpp/processors/tests/test_sixdee.py @@ -70,8 +70,10 @@ def setUp(self): } @inlineCallbacks - def get_transport(self, config={}, bind=True): + def get_transport(self, deliver_config={}, submit_config={}, bind=True): cfg = self.default_config.copy() + cfg['deliver_short_message_processor_config'].update(deliver_config) + cfg['submit_short_message_processor_config'].update(submit_config) transport = yield self.tx_helper.get_transport(cfg, start=False) transport.clock = self.clock yield transport.startWorker() @@ -149,6 +151,36 @@ def test_submit_and_deliver_ussd_new(self): } }) + @inlineCallbacks + def test_submit_and_deliver_ussd_new_custom_ussd_code_field(self): + session = SessionInfo() + yield self.get_transport(deliver_config={ + 'ussd_code_pdu_field': 'destination_addr', + }) + + # Server delivers a USSD message to the Client + pdu = DeliverSM(1, short_message="*IGNORE#", destination_addr="*123#") + pdu.add_optional_parameter('ussd_service_op', '01') + pdu.add_optional_parameter('its_session_info', session.its_info) + + yield self.fake_smsc.handle_pdu(pdu) + + [mess] = yield self.tx_helper.wait_for_dispatched_inbound(1) + + self.assertEqual(mess['content'], None) + self.assertEqual(mess['to_addr'], '*123#') + self.assertEqual(mess['transport_type'], "ussd") + self.assertEqual(mess['session_event'], + TransportUserMessage.SESSION_NEW) + self.assertEqual( + mess['transport_metadata'], + { + 'session_info': { + 'session_identifier': session.sixdee_id, + 'ussd_service_op': '01', + } + }) + @inlineCallbacks def test_deliver_sm_op_codes_new(self): session = SessionInfo() From e381db58b0883933f7bf01ec634525afaf963e3b Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Mon, 25 Jul 2016 17:28:00 +0200 Subject: [PATCH 14/25] Avoid use of lambda to make flake8 happy. --- vumi/transports/smpp/processors/tests/test_sixdee.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/vumi/transports/smpp/processors/tests/test_sixdee.py b/vumi/transports/smpp/processors/tests/test_sixdee.py index ac03d82a6..ea184990a 100644 --- a/vumi/transports/smpp/processors/tests/test_sixdee.py +++ b/vumi/transports/smpp/processors/tests/test_sixdee.py @@ -83,9 +83,15 @@ def get_transport(self, deliver_config={}, submit_config={}, bind=True): returnValue(transport) def assert_udh_parts(self, pdus, texts, encoding): - pdu_header = lambda pdu: short_message(pdu)[:6] - pdu_text = lambda pdu: short_message(pdu)[6:].decode(encoding) - udh_header = lambda i: '\x05\x00\x03\x03\x07' + chr(i) + def pdu_header(pdu): + return short_message(pdu)[:6] + + def pdu_text(pdu): + return short_message(pdu)[6:].decode(encoding) + + def udh_header(i): + return '\x05\x00\x03\x03\x07' + chr(i) + self.assertEqual( [(pdu_header(pdu), pdu_text(pdu)) for pdu in pdus], [(udh_header(i + 1), text) for i, text in enumerate(texts)]) From 1c01468d385c9aa4005d83cc9f9e5b06eccece65 Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Tue, 26 Jul 2016 09:50:36 +0200 Subject: [PATCH 15/25] Fix Mixit tests. --- vumi/transports/mxit/tests/test_mxit.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/vumi/transports/mxit/tests/test_mxit.py b/vumi/transports/mxit/tests/test_mxit.py index 031299bd8..eaaaf3be9 100644 --- a/vumi/transports/mxit/tests/test_mxit.py +++ b/vumi/transports/mxit/tests/test_mxit.py @@ -2,7 +2,7 @@ import base64 from twisted.internet.defer import inlineCallbacks, DeferredQueue -from twisted.web.http import Request, BAD_REQUEST +from twisted.web.http import BAD_REQUEST from twisted.web.server import NOT_DONE_YET from vumi.transports.mxit import MxitTransport @@ -10,6 +10,7 @@ from vumi.utils import http_request_full from vumi.tests.helpers import VumiTestCase from vumi.tests.fake_connection import FakeHttpServer +from twisted.web.test.requesthelper import DummyRequest from vumi.transports.tests.helpers import TransportHelper @@ -62,7 +63,7 @@ def handle_request(self, request): return NOT_DONE_YET def test_is_mxit_request(self): - req = Request(None, True) + req = DummyRequest([]) self.assertFalse(self.transport.is_mxit_request(req)) req.requestHeaders.addRawHeader('X-Mxit-Contact', 'foo') self.assertTrue(self.transport.is_mxit_request(req)) @@ -100,7 +101,7 @@ def test_html_decode(self): self.transport.html_decode(self.sample_html_str), '<&>') def test_get_request_data(self): - req = Request(None, True) + req = DummyRequest([]) headers = req.requestHeaders for key, value in self.sample_req_headers.items(): headers.addRawHeader(key, value) @@ -134,29 +135,29 @@ def test_get_request_data(self): }) def test_get_request_content_from_header(self): - req = Request(None, True) + req = DummyRequest([]) req.requestHeaders.addRawHeader('X-Mxit-User-Input', 'foo') self.assertEqual(self.transport.get_request_content(req), 'foo') def test_get_quote_plus_request_content_from_header(self): - req = Request(None, True) + req = DummyRequest([]) req.requestHeaders.addRawHeader('X-Mxit-User-Input', 'foo+bar') self.assertEqual( self.transport.get_request_content(req), 'foo bar') def test_get_quoted_request_content_from_header(self): - req = Request(None, True) + req = DummyRequest([]) req.requestHeaders.addRawHeader('X-Mxit-User-Input', 'foo%20bar') self.assertEqual( self.transport.get_request_content(req), 'foo bar') def test_get_request_content_from_args(self): - req = Request(None, True) + req = DummyRequest([]) req.args = {'input': ['bar']} self.assertEqual(self.transport.get_request_content(req), 'bar') def test_get_request_content_when_missing(self): - req = Request(None, True) + req = DummyRequest([]) self.assertEqual(self.transport.get_request_content(req), None) @inlineCallbacks From 4eb34951c1cbb696290cf09c30290c568789a6c0 Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Tue, 26 Jul 2016 14:05:24 +0200 Subject: [PATCH 16/25] Only return decimal digits in the session id. --- vumi/transports/mtn_nigeria/xml_over_tcp.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/vumi/transports/mtn_nigeria/xml_over_tcp.py b/vumi/transports/mtn_nigeria/xml_over_tcp.py index 5e0734d8e..417f407fd 100644 --- a/vumi/transports/mtn_nigeria/xml_over_tcp.py +++ b/vumi/transports/mtn_nigeria/xml_over_tcp.py @@ -1,6 +1,5 @@ -import uuid +import random import struct -from random import randint from twisted.web import microdom from twisted.internet import reactor @@ -58,6 +57,7 @@ class XmlOverTcpClient(Protocol): LENGTH_HEADER_SIZE = 16 HEADER_SIZE = SESSION_ID_HEADER_SIZE + LENGTH_HEADER_SIZE HEADER_FORMAT = '!%ss%ss' % (SESSION_ID_HEADER_SIZE, LENGTH_HEADER_SIZE) + SESSION_ID_CHARACTERS = "0123456789" REQUEST_ID_LENGTH = 10 @@ -407,17 +407,16 @@ def gen_session_id(cls): """ Generates session id. Used for packets needing a dummy session id. """ - # NOTE: Slicing the generated uuid is probably a bad idea, and will - # affect collision resistence, but I can't think of a simpler way to - # generate a unique 16 char alphanumeric. - return uuid.uuid4().hex[:cls.SESSION_ID_HEADER_SIZE] + return "".join( + random.choice(cls.SESSION_ID_CHARACTERS) + for i in range(cls.SESSION_ID_HEADER_SIZE)) @classmethod def gen_request_id(cls): # NOTE: The protocol requires request ids to be number only ids. With a # request id length of 10 digits, generating ids using randint could # well cause collisions to occur, although this should be unlikely. - return str(randint(0, (10 ** cls.REQUEST_ID_LENGTH) - 1)) + return str(random.randint(0, (10 ** cls.REQUEST_ID_LENGTH) - 1)) def login(self): params = [ From f1282f52726bbd9cb0edf7c50e9e70f6f0273d93 Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Tue, 26 Jul 2016 14:09:11 +0200 Subject: [PATCH 17/25] Add test for gen_session_id. --- vumi/transports/mtn_nigeria/tests/test_xml_over_tcp.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vumi/transports/mtn_nigeria/tests/test_xml_over_tcp.py b/vumi/transports/mtn_nigeria/tests/test_xml_over_tcp.py index 42e82f487..f3145e63b 100644 --- a/vumi/transports/mtn_nigeria/tests/test_xml_over_tcp.py +++ b/vumi/transports/mtn_nigeria/tests/test_xml_over_tcp.py @@ -856,3 +856,9 @@ def test_error_response_handling_for_unknown_codes(self): 'err', "Server sent error message: (1337) Unknown Code: " "Some Reason") + + def test_gen_session_id(self): + sessid = XmlOverTcpClient.gen_session_id() + self.assertEqual(len(sessid), XmlOverTcpClient.SESSION_ID_HEADER_SIZE) + self.assertTrue( + all(c in XmlOverTcpClient.SESSION_ID_CHARACTERS for c in sessid)) From 07633c3b10e539766df327e982d7c0f222f611d9 Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Tue, 26 Jul 2016 14:41:29 +0200 Subject: [PATCH 18/25] Check request.connection_has_been_closed before writing streaming responses. --- vumi/components/message_store_resource.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vumi/components/message_store_resource.py b/vumi/components/message_store_resource.py index 8e05b09bb..b4d614024 100644 --- a/vumi/components/message_store_resource.py +++ b/vumi/components/message_store_resource.py @@ -155,7 +155,8 @@ def handle_message(self, message_key, request): return d def write_message(self, message, request): - self.formatter.write_row(request, message) + if not request.connection_has_been_closed: + self.formatter.write_row(request, message) class InboundResource(MessageStoreProxyResource): From 32321fdf9b56283f13b0679826406b2bc34e15e7 Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Tue, 26 Jul 2016 15:46:35 +0200 Subject: [PATCH 19/25] Correctly handle requests that are already closed. --- vumi/transports/trueafrican/transport.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/vumi/transports/trueafrican/transport.py b/vumi/transports/trueafrican/transport.py index da08c9407..45a8e1d96 100644 --- a/vumi/transports/trueafrican/transport.py +++ b/vumi/transports/trueafrican/transport.py @@ -6,18 +6,24 @@ import collections -from twisted.internet.defer import Deferred, returnValue, inlineCallbacks +from twisted.internet.defer import Deferred, returnValue, inlineCallbacks, fail from twisted.internet.task import LoopingCall from twisted.internet import reactor +from twisted.python.failure import Failure from twisted.web import xmlrpc, server from vumi import log +from vumi.errors import VumiError from vumi.message import TransportUserMessage from vumi.transports.base import Transport from vumi.components.session import SessionManager from vumi.config import ConfigText, ConfigInt, ConfigDict +class TrueAfricanError(VumiError): + """Raised by errors in the TrueAfrican transport.""" + + class TrueAfricanUssdTransportConfig(Transport.CONFIG_CLASS): """TrueAfrican USSD transport configuration.""" @@ -221,7 +227,12 @@ def finish_request(self, request_id, message_id, response): # Add a callback and errback, either of which will be invoked # depending on whether the response was written to the client # successfully or not - request.http_request.notifyFinish().addCallbacks( + if request.http_request.content.closed: + request_done = fail(Failure(TrueAfricanError( + "HTTP client closed connection"))) + else: + request_done = request.http_request.notifyFinish() + request_done.addCallbacks( lambda _: self._finish_success_cb(message_id), lambda f: self._finish_failure_cb(f, message_id) ) From f171301ac952faecab551567f465084b55110724 Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Tue, 26 Jul 2016 15:47:13 +0200 Subject: [PATCH 20/25] Use request.content.closed to check for closed connection as more consistently available. --- vumi/components/message_store_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vumi/components/message_store_resource.py b/vumi/components/message_store_resource.py index b4d614024..7a0b3924a 100644 --- a/vumi/components/message_store_resource.py +++ b/vumi/components/message_store_resource.py @@ -155,7 +155,7 @@ def handle_message(self, message_key, request): return d def write_message(self, message, request): - if not request.connection_has_been_closed: + if not request.content.closed: self.formatter.write_row(request, message) From c84d2ed1584b84bc5ba98019ecb52341dd945e46 Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Tue, 26 Jul 2016 15:55:09 +0200 Subject: [PATCH 21/25] Add check for nack_reason message. --- vumi/transports/trueafrican/tests/test_transport.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vumi/transports/trueafrican/tests/test_transport.py b/vumi/transports/trueafrican/tests/test_transport.py index 62ce2d3d5..5317d99e4 100644 --- a/vumi/transports/trueafrican/tests/test_transport.py +++ b/vumi/transports/trueafrican/tests/test_transport.py @@ -213,6 +213,7 @@ def test_nack_for_outbound_message(self): self.assertEqual(nack['event_type'], 'nack') self.assertEqual(nack['user_message_id'], rep['message_id']) self.assertEqual(nack['sent_message_id'], rep['message_id']) + self.assertTrue('HTTP client closed connection' in nack['nack_reason']) @inlineCallbacks def test_nack_for_request_timeout(self): From 2c881c109fef0b654ae5e23e9c6335c11eb75943 Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Tue, 26 Jul 2016 17:09:27 +0200 Subject: [PATCH 22/25] Add .requestHeaders to older versions of DummyRequest. --- vumi/transports/mxit/tests/test_mxit.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/vumi/transports/mxit/tests/test_mxit.py b/vumi/transports/mxit/tests/test_mxit.py index eaaaf3be9..4b47723ae 100644 --- a/vumi/transports/mxit/tests/test_mxit.py +++ b/vumi/transports/mxit/tests/test_mxit.py @@ -4,16 +4,25 @@ from twisted.internet.defer import inlineCallbacks, DeferredQueue from twisted.web.http import BAD_REQUEST from twisted.web.server import NOT_DONE_YET +from twisted.web.http_headers import Headers from vumi.transports.mxit import MxitTransport from vumi.transports.mxit.responses import ResponseParser from vumi.utils import http_request_full from vumi.tests.helpers import VumiTestCase from vumi.tests.fake_connection import FakeHttpServer -from twisted.web.test.requesthelper import DummyRequest +from twisted.web.test.requesthelper import DummyRequest as TwistedDummyRequest from vumi.transports.tests.helpers import TransportHelper +class DummyRequest(TwistedDummyRequest): + def __init__(self, *args, **kw): + # Twisted 13.2.0 doesn't have .requestHeaders on DummyRequest + TwistedDummyRequest.__init__(self, *args, **kw) + if not hasattr(self, 'requestHeaders'): + self.requestHeaders = Headers() + + class TestMxitTransport(VumiTestCase): @inlineCallbacks From 3aa4dcd1bec789b379ad1a901adfcc23bc704461 Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Tue, 26 Jul 2016 17:30:38 +0200 Subject: [PATCH 23/25] Update release notes. --- docs/release-notes.rst | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/release-notes.rst b/docs/release-notes.rst index f421c4009..9e8f09a21 100644 --- a/docs/release-notes.rst +++ b/docs/release-notes.rst @@ -11,12 +11,22 @@ this will almost certainly not break the majority of things built on vumi, old code or code that relies too heavily on the details of worker setup may need to be fixed. +:Version: 0.6.9 +:Date released: 27 July 2016 + +* Apply numerous cleanups to the Dockerfile. +* Use only decimal digits for session identifiers in the MTN Nigeria USSD + XML over TCP transport. +* Add the ability to configure the PDU field the dialed USSD code is taken + from in the 6D SMPP processor. +* Update tests to pass with Twisted 16.3. + :Version: 0.6.8 :Date released: 12 May 2016 -* Allow disabling of delivery report handling as sometimes these cause more noise +* Allow disabling of delivery report handling as sometimes these cause more noise than signal. -* Embed the original SMPP transports delivery report status into the message +* Embed the original SMPP transports delivery report status into the message transport metadata. This is useful information that applications may chose to act on. From 43980e4bd2437e7c2174780802a7e2ddece22b2c Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Tue, 26 Jul 2016 17:39:10 +0200 Subject: [PATCH 24/25] Add updating Dockerfile version to version bumping script. --- utils/bump-version.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bump-version.sh b/utils/bump-version.sh index fa0bba486..1077a827b 100755 --- a/utils/bump-version.sh +++ b/utils/bump-version.sh @@ -22,3 +22,4 @@ inplace_sed "s/\(__version__[ ]*=[ ]*[\"']\)\(.*\)\([\"'].*\)/\1${VER}\3/" vumi/ inplace_sed "s/\(version[ ]*=[ ]*[\"']\)\(.*\)\([\"'].*\)/\1${VER}\3/" setup.py inplace_sed "s/^\(release[ ]*=[ ]*[\"']\)\(.*\)\([\"'].*\)/\1${VER}\3/" docs/conf.py inplace_sed "s/^\(version[ ]*=[ ]*[\"']\)\(.*\)\([\"'].*\)/\1${SHORT_VER}\3/" docs/conf.py +inplace_sed "s/^\(ENV [ ]*VUMI_VERSION [ ]*\"\)\([^\"]*\)\(\".*\)/\1${VER}\3/" docker/Dockerfile From 889d28da4bb4adc050aa5eb8658df7b17e3464a1 Mon Sep 17 00:00:00 2001 From: Simon Cross Date: Wed, 27 Jul 2016 09:40:14 +0200 Subject: [PATCH 25/25] Version 0.6.9 --- docker/Dockerfile | 2 +- docs/conf.py | 2 +- setup.py | 2 +- vumi/__init__.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index db7b198d1..9d196b7a6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ FROM praekeltfoundation/python-base MAINTAINER Praekelt Foundation -ENV VUMI_VERSION "0.6.8" +ENV VUMI_VERSION "0.6.9" RUN pip install vumi==$VUMI_VERSION COPY ./vumi-entrypoint.sh /app/vumi-entrypoint.sh diff --git a/docs/conf.py b/docs/conf.py index b229a2c65..602100c22 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -53,7 +53,7 @@ # The short X.Y version. version = '0.6' # The full version, including alpha/beta/rc tags. -release = '0.6.8' +release = '0.6.9' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index d3f00101c..cf9a7aee2 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name="vumi", - version="0.6.8", + version="0.6.9", url='http://github.com/praekelt/vumi', license='BSD', description="Super-scalable messaging engine for the delivery of SMS, " diff --git a/vumi/__init__.py b/vumi/__init__.py index c8bdce6ea..5c7142f51 100644 --- a/vumi/__init__.py +++ b/vumi/__init__.py @@ -2,4 +2,4 @@ Vumi scalable text messaging engine. """ -__version__ = "0.6.8" +__version__ = "0.6.9"