Merge pull request #745 from alphagov/pc-nab-numerous-notifications

[#131977833] Create get all notifications endpoint for V2 API

app/dao/notifications_dao.py

@@ -262,7 +262,8 @@ def get_notifications_for_service(
     key_type=None,
     personalisation=False,
     include_jobs=False,
-    include_from_test_key=False
+    include_from_test_key=False,
+    older_than=None
 ):
     if page_size is None:
         page_size = current_app.config['PAGE_SIZE']
@@ -273,6 +274,11 @@ def get_notifications_for_service(
         days_ago = date.today() - timedelta(days=limit_days)
         filters.append(func.date(Notification.created_at) >= days_ago)
 
+    if older_than is not None:
+        older_than_created_at = db.session.query(
+            Notification.created_at).filter(Notification.id == older_than).as_scalar()
+        filters.append(Notification.created_at < older_than_created_at)
+
     if not include_jobs or (key_type and key_type != KEY_TYPE_NORMAL):
         filters.append(Notification.job_id.is_(None))
 
@@ -296,15 +302,21 @@ def get_notifications_for_service(
 
 def _filter_query(query, filter_dict=None):
     if filter_dict is None:
-        filter_dict = MultiDict()
-    else:
-        filter_dict = MultiDict(filter_dict)
-    statuses = filter_dict.getlist('status') if 'status' in filter_dict else None
+        return query
+
+    multidict = MultiDict(filter_dict)
+
+    # filter by status
+    statuses = multidict.getlist('status')
     if statuses:
+        statuses = Notification.substitute_status(statuses)
         query = query.filter(Notification.status.in_(statuses))
-    template_types = filter_dict.getlist('template_type') if 'template_type' in filter_dict else None
+
+    # filter by template
+    template_types = multidict.getlist('template_type')
     if template_types:
         query = query.join(Template).filter(Template.template_type.in_(template_types))
+
     return query
 
 

app/models.py

@@ -6,7 +6,7 @@
     UUID,
     JSON
 )
-from sqlalchemy import UniqueConstraint, and_, desc
+from sqlalchemy import UniqueConstraint, and_
 from sqlalchemy.orm import foreign, remote
 from notifications_utils.recipients import (
     validate_email_address,
@@ -178,9 +178,8 @@ def from_string(cls, service_id, recipient_type, recipient):
         else:
             return instance
 
-        def __repr__(self):
-            return 'Recipient {} of type: {}'.format(self.recipient,
-                                                     self.recipient_type)
+    def __repr__(self):
+        return 'Recipient {} of type: {}'.format(self.recipient, self.recipient_type)
 
 
 class ApiKey(db.Model, Versioned):
@@ -285,7 +284,12 @@ class Template(db.Model):
 
     def get_link(self):
         # TODO: use "/v2/" route once available
-        return url_for("template.get_template_by_id_and_service_id", service_id=self.service_id, template_id=self.id)
+        return url_for(
+            "template.get_template_by_id_and_service_id",
+            service_id=self.service_id,
+            template_id=self.id,
+            _external=True
+        )
 
 
 class TemplateHistory(db.Model):
@@ -466,13 +470,27 @@ def check_code(self, cde):
 NOTIFICATION_TEMPORARY_FAILURE = 'temporary-failure'
 NOTIFICATION_PERMANENT_FAILURE = 'permanent-failure'
 
+NOTIFICATION_STATUS_TYPES_FAILED = [
+    NOTIFICATION_TECHNICAL_FAILURE,
+    NOTIFICATION_TEMPORARY_FAILURE,
+    NOTIFICATION_PERMANENT_FAILURE,
+]
+
+NOTIFICATION_STATUS_TYPES_COMPLETED = [
+    NOTIFICATION_DELIVERED,
+    NOTIFICATION_FAILED,
+    NOTIFICATION_TECHNICAL_FAILURE,
+    NOTIFICATION_TEMPORARY_FAILURE,
+    NOTIFICATION_PERMANENT_FAILURE,
+]
+
 NOTIFICATION_STATUS_TYPES_BILLABLE = [
     NOTIFICATION_SENDING,
     NOTIFICATION_DELIVERED,
     NOTIFICATION_FAILED,
     NOTIFICATION_TECHNICAL_FAILURE,
     NOTIFICATION_TEMPORARY_FAILURE,
-    NOTIFICATION_PERMANENT_FAILURE
+    NOTIFICATION_PERMANENT_FAILURE,
 ]
 
 NOTIFICATION_STATUS_TYPES = [
@@ -483,7 +501,7 @@ def check_code(self, cde):
     NOTIFICATION_FAILED,
     NOTIFICATION_TECHNICAL_FAILURE,
     NOTIFICATION_TEMPORARY_FAILURE,
-    NOTIFICATION_PERMANENT_FAILURE
+    NOTIFICATION_PERMANENT_FAILURE,
 ]
 NOTIFICATION_STATUS_TYPES_ENUM = db.Enum(*NOTIFICATION_STATUS_TYPES, name='notify_status_type')
 
@@ -544,33 +562,52 @@ def personalisation(self, personalisation):
         if personalisation:
             self._personalisation = encryption.encrypt(personalisation)
 
-    def cost(self):
-        if not self.sent_by or self.billable_units == 0:
-            return 0
-
-        provider_rate = db.session.query(
-            ProviderRates
-        ).join(ProviderDetails).filter(
-            ProviderDetails.identifier == self.sent_by,
-            ProviderRates.provider_id == ProviderDetails.id
-        ).order_by(
-            desc(ProviderRates.valid_from)
-        ).limit(1).one()
-
-        return float(provider_rate.rate * self.billable_units)
-
     def completed_at(self):
-        if self.status in [
-            NOTIFICATION_DELIVERED,
-            NOTIFICATION_FAILED,
-            NOTIFICATION_TECHNICAL_FAILURE,
-            NOTIFICATION_TEMPORARY_FAILURE,
-            NOTIFICATION_PERMANENT_FAILURE
-        ]:
+        if self.status in NOTIFICATION_STATUS_TYPES_COMPLETED:
             return self.updated_at.strftime(DATETIME_FORMAT)
 
         return None
 
+    @staticmethod
+    def substitute_status(status_or_statuses):
+        """
+        static function that takes a status or list of statuses and substitutes our new failure types if it finds
+        the deprecated one
+
+        > IN
+        'failed'
+
+        < OUT
+        ['technical-failure', 'temporary-failure', 'permanent-failure']
+
+        -
+
+        > IN
+        ['failed', 'created']
+
+        < OUT
+        ['technical-failure', 'temporary-failure', 'permanent-failure', 'created']
+
+
+        :param status_or_statuses: a single status or list of statuses
+        :return: a single status or list with the current failure statuses substituted for 'failure'
+        """
+
+        def _substitute_status_str(_status):
+            return NOTIFICATION_STATUS_TYPES_FAILED if _status == NOTIFICATION_FAILED else _status
+
+        def _substitute_status_seq(_statuses):
+            if NOTIFICATION_FAILED in _statuses:
+                _statuses = list(set(
+                    NOTIFICATION_STATUS_TYPES_FAILED + [_s for _s in _statuses if _s != NOTIFICATION_FAILED]
+                ))
+            return _statuses
+
+        if isinstance(status_or_statuses, str):
+            return _substitute_status_str(status_or_statuses)
+
+        return _substitute_status_seq(status_or_statuses)
+
     def serialize(self):
 
         template_dict = {
@@ -591,7 +628,6 @@ def serialize(self):
             "line_5": None,
             "line_6": None,
             "postcode": None,
-            "cost": self.cost(),
             "type": self.notification_type,
             "status": self.status,
             "template": template_dict,

app/notifications/rest.py

@@ -170,8 +170,8 @@ def get_notification_by_id(notification_id):
 def get_all_notifications():
     data = notifications_filter_schema.load(request.args).data
     include_jobs = data.get('include_jobs', False)
-    page = data['page'] if 'page' in data else 1
-    page_size = data['page_size'] if 'page_size' in data else current_app.config.get('PAGE_SIZE')
+    page = data.get('page', 1)
+    page_size = data.get('page_size', current_app.config.get('PAGE_SIZE'))
     limit_days = data.get('limit_days')
 
     pagination = notifications_dao.get_notifications_for_service(

app/schemas.py

@@ -468,6 +468,7 @@ class Meta:
     limit_days = fields.Int(required=False)
     include_jobs = fields.Boolean(required=False)
     include_from_test_key = fields.Boolean(required=False)
+    older_than = fields.UUID(required=False)
 
     @pre_load
     def handle_multidict(self, in_data):

app/v2/notifications/get_notifications.py

@@ -1,7 +1,8 @@
-from flask import jsonify
+from flask import jsonify, request, url_for
 
 from app import api_user
 from app.dao import notifications_dao
+from app.schemas import notifications_filter_schema
 from app.v2.notifications import notification_blueprint
 
 
@@ -14,9 +15,30 @@ def get_notification_by_id(id):
     return jsonify(notification.serialize()), 200
 
 
-@notification_blueprint.route("/", methods=['GET'])
+@notification_blueprint.route("", methods=['GET'])
 def get_notifications():
-    # validate notifications request arguments
-    # fetch all notifications
-    # return notifications_response schema
-    pass
+    data = notifications_filter_schema.load(request.args).data
+
+    paginated_notifications = notifications_dao.get_notifications_for_service(
+        str(api_user.service_id),
+        filter_dict=data,
+        key_type=api_user.key_type,
+        personalisation=True,
+        older_than=data.get('older_than')
+    )
+
+    def _build_links(notifications):
+        _links = {
+            'current': url_for(".get_notifications", _external=True, **request.args.to_dict(flat=False)),
+        }
+
+        if len(notifications):
+            next_query_params = dict(request.args.to_dict(flat=False), older_than=notifications[-1].id)
+            _links['next'] = url_for(".get_notifications", _external=True, **next_query_params)
+
+        return _links
+
+    return jsonify(
+        notifications=[notification.serialize() for notification in paginated_notifications.items],
+        links=_build_links(paginated_notifications.items)
+    ), 200

app/v2/notifications/notification_schemas.py

@@ -57,7 +57,6 @@
         "line_5": {"type": ["string", "null"]},
         "line_6": {"type": ["string", "null"]},
         "postcode": {"type": ["string", "null"]},
-        "cost": {"type": "number"},
         "type": {"enum": ["sms", "letter", "email"]},
         "status": {"type": "string"},
         "template": template,
@@ -69,11 +68,40 @@
         # technically, all keys are required since we always have all of them
         "id", "reference", "email_address", "phone_number",
         "line_1", "line_2", "line_3", "line_4", "line_5", "line_6", "postcode",
-        "cost", "type", "status", "template",
-        "created_at", "sent_at", "completed_at"
+        "type", "status", "template", "created_at", "sent_at", "completed_at"
     ]
 }
 
+get_notifications_response = {
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "GET list of notifications response schema",
+    "type": "object",
+    "properties": {
+        "notifications": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "ref": get_notification_response
+            }
+        },
+        "links": {
+            "type": "object",
+            "properties": {
+                "current": {
+                    "type": "string"
+                },
+                "next": {
+                    "type": "string"
+                }
+            },
+            "additionalProperties": False,
+            "required": ["current"]
+        }
+    },
+    "additionalProperties": False,
+    "required": ["notifications", "links"]
+}
+
 post_sms_request = {
     "$schema": "http://json-schema.org/draft-04/schema#",
     "description": "POST sms notification schema",

tests/app/conftest.py

@@ -21,7 +21,7 @@
     NotificationStatistics,
     ServiceWhitelist,
     KEY_TYPE_NORMAL, KEY_TYPE_TEST, KEY_TYPE_TEAM,
-    MOBILE_TYPE, EMAIL_TYPE)
+    MOBILE_TYPE, EMAIL_TYPE, NOTIFICATION_STATUS_TYPES_COMPLETED)
 from app.dao.users_dao import (save_model_user, create_user_code, create_secret_code)
 from app.dao.services_dao import (dao_create_service, dao_add_user_to_service)
 from app.dao.templates_dao import dao_create_template
@@ -444,7 +444,8 @@ def sample_notification(notify_db,
         'notification_type': template.template_type,
         'api_key_id': api_key_id,
         'key_type': key_type,
-        'sent_by': sent_by
+        'sent_by': sent_by,
+        'updated_at': created_at if status in NOTIFICATION_STATUS_TYPES_COMPLETED else None
     }
     if job_row_number:
         data['job_row_number'] = job_row_number

tests/app/dao/test_notification_dao.py

@@ -8,15 +8,14 @@
 from freezegun import freeze_time
 from sqlalchemy.exc import SQLAlchemyError, IntegrityError
 
-from app import db
-
 from app.models import (
     Notification,
     NotificationHistory,
     Job,
     NotificationStatistics,
     TemplateStatistics,
     NOTIFICATION_STATUS_TYPES,
+    NOTIFICATION_STATUS_TYPES_FAILED,
     KEY_TYPE_NORMAL,
     KEY_TYPE_TEAM,
     KEY_TYPE_TEST
@@ -683,7 +682,10 @@ def test_get_all_notifications_for_job_by_status(notify_db, notify_db_session, s
     assert len(notifications().items) == len(NOTIFICATION_STATUS_TYPES)
 
     for status in NOTIFICATION_STATUS_TYPES:
-        assert len(notifications(filter_dict={'status': status}).items) == 1
+        if status == 'failed':
+            assert len(notifications(filter_dict={'status': status}).items) == len(NOTIFICATION_STATUS_TYPES_FAILED)
+        else:
+            assert len(notifications(filter_dict={'status': status}).items) == 1
 
     assert len(notifications(filter_dict={'status': NOTIFICATION_STATUS_TYPES[:3]}).items) == 3