Merge branch 'master' into research-mode-csv-file-queues

Conflicts:
	tests/app/celery/test_tasks.py
	tests/app/notifications/rest/test_send_notification.py

app/celery/tasks.py

@@ -1,10 +1,8 @@
-import itertools
 from datetime import (datetime)
 
 from flask import current_app
 from notifications_utils.recipients import (
-    RecipientCSV,
-    allowed_to_send_to
+    RecipientCSV
 )
 from notifications_utils.template import Template
 from sqlalchemy.exc import SQLAlchemyError
@@ -31,6 +29,7 @@
     KEY_TYPE_NORMAL,
     KEY_TYPE_TEST
 )
+from app.service.utils import service_allowed_to_send_to
 from app.statsd_decorators import statsd
 
 
@@ -188,15 +187,3 @@ def send_email(self, service_id,
                 "RETRY FAILED: task send_email failed for notification {}".format(notification.id),
                 e
             )
-
-
-def service_allowed_to_send_to(recipient, service, key_type):
-    if not service.restricted or key_type == KEY_TYPE_TEST:
-        return True
-
-    return allowed_to_send_to(
-        recipient,
-        itertools.chain.from_iterable(
-            [user.mobile_number, user.email_address] for user in service.users
-        )
-    )

app/dao/dao_utils.py

@@ -1,6 +1,7 @@
 import itertools
 from functools import wraps, partial
 
+from app import db
 from app.history_meta import create_history
 
 
@@ -35,3 +36,7 @@ def record_version(*args, **kwargs):
                 db.session.add(h_obj)
         return record_version
     return versioned
+
+
+def dao_rollback():
+    db.session.rollback()

app/dao/service_whitelist_dao.py

@@ -0,0 +1,17 @@
+from app import db
+from app.models import Service, ServiceWhitelist
+
+
+def dao_fetch_service_whitelist(service_id):
+    return ServiceWhitelist.query.filter(
+        ServiceWhitelist.service_id == service_id).all()
+
+
+def dao_add_and_commit_whitelisted_contacts(objs):
+    db.session.add_all(objs)
+    db.session.commit()
+
+
+def dao_remove_service_whitelist(service_id):
+    return ServiceWhitelist.query.filter(
+        ServiceWhitelist.service_id == service_id).delete()

app/models.py

@@ -1,11 +1,18 @@
 import uuid
 import datetime
+
 from sqlalchemy.dialects.postgresql import (
     UUID,
     JSON
 )
-from sqlalchemy import UniqueConstraint, text, ForeignKeyConstraint, and_
+from sqlalchemy import UniqueConstraint, and_
 from sqlalchemy.orm import foreign, remote
+from notifications_utils.recipients import (
+    validate_email_address,
+    validate_phone_number,
+    InvalidPhoneError,
+    InvalidEmailError
+)
 
 from app.encryption import (
     hashpw,
@@ -131,6 +138,47 @@ class Service(db.Model, Versioned):
         default=BRANDING_GOVUK
     )
 
+MOBILE_TYPE = 'mobile'
+EMAIL_TYPE = 'email'
+
+WHITELIST_RECIPIENT_TYPE = [MOBILE_TYPE, EMAIL_TYPE]
+whitelist_recipient_types = db.Enum(*WHITELIST_RECIPIENT_TYPE, name='recipient_type')
+
+
+class ServiceWhitelist(db.Model):
+    __tablename__ = 'service_whitelist'
+
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False)
+    service = db.relationship('Service', backref='whitelist')
+    recipient_type = db.Column(whitelist_recipient_types, nullable=False)
+    recipient = db.Column(db.String(255), nullable=False)
+    created_at = db.Column(db.DateTime, default=datetime.datetime.utcnow)
+
+    @classmethod
+    def from_string(cls, service_id, recipient_type, recipient):
+        instance = cls(service_id=service_id, recipient_type=recipient_type)
+
+        try:
+            if recipient_type == MOBILE_TYPE:
+                validate_phone_number(recipient)
+                instance.recipient = recipient
+            elif recipient_type == EMAIL_TYPE:
+                validate_email_address(recipient)
+                instance.recipient = recipient
+            else:
+                raise ValueError('Invalid recipient type')
+        except InvalidPhoneError:
+            raise ValueError('Invalid whitelist: "{}"'.format(recipient))
+        except InvalidEmailError:
+            raise ValueError('Invalid whitelist: "{}"'.format(recipient))
+        else:
+            return instance
+
+        def __repr__(self):
+            return 'Recipient {} of type: {}'.format(self.recipient,
+                                                     self.recipient_type)
+
 
 class ApiKey(db.Model, Versioned):
     __tablename__ = 'api_keys'

app/notifications/rest.py

@@ -9,7 +9,7 @@
     json
 )
 
-from notifications_utils.recipients import allowed_to_send_to, first_column_heading
+from notifications_utils.recipients import first_column_heading
 from notifications_utils.template import Template
 from notifications_utils.renderers import PassThrough
 from app.clients.email.aws_ses import get_aws_responses
@@ -27,6 +27,7 @@
     validate_callback_data,
     process_sms_client_response
 )
+from app.service.utils import service_allowed_to_send_to
 from app.schemas import (
     email_notification_schema,
     sms_template_notification_schema,
@@ -252,16 +253,7 @@ def send_notification(notification_type):
         errors = {'content': [message]}
         raise InvalidRequest(errors, status_code=400)
 
-    if all((
-        api_user.key_type != KEY_TYPE_TEST,
-        service.restricted or api_user.key_type == KEY_TYPE_TEAM,
-        not allowed_to_send_to(
-            notification['to'],
-            itertools.chain.from_iterable(
-                [user.mobile_number, user.email_address] for user in service.users
-            )
-        )
-    )):
+    if not service_allowed_to_send_to(notification['to'], service, api_user.key_type):
         if (api_user.key_type == KEY_TYPE_TEAM):
             message = 'Can’t send to this recipient using a team-only API key'
         else:
@@ -276,7 +268,6 @@ def send_notification(notification_type):
 
     notification_id = create_uuid()
     notification.update({"template_version": template.version})
-
     if not _simulated_recipient(notification['to'], notification_type):
         persist_notification(
             service,

app/service/rest.py

@@ -8,6 +8,7 @@
 )
 from sqlalchemy.orm.exc import NoResultFound
 
+from app.dao.dao_utils import dao_rollback
 from app.dao.api_key_dao import (
     save_model_api_key,
     get_model_api_keys,
@@ -26,9 +27,20 @@
     dao_fetch_weekly_historical_stats_for_service,
     dao_fetch_todays_stats_for_all_services
 )
+from app.dao.service_whitelist_dao import (
+    dao_fetch_service_whitelist,
+    dao_add_and_commit_whitelisted_contacts,
+    dao_remove_service_whitelist
+)
 from app.dao import notifications_dao
 from app.dao.provider_statistics_dao import get_fragment_count
 from app.dao.users_dao import get_model_users
+from app.errors import (
+    register_errors,
+    InvalidRequest
+)
+from app.service import statistics
+from app.service.utils import get_whitelist_objects
 from app.schemas import (
     service_schema,
     api_key_schema,
@@ -39,11 +51,6 @@
     detailed_service_schema
 )
 from app.utils import pagination_links
-from app.errors import (
-    register_errors,
-    InvalidRequest
-)
-from app.service import statistics
 
 service_blueprint = Blueprint('service', __name__)
 register_errors(service_blueprint)
@@ -270,3 +277,36 @@ def get_detailed_services():
             service.statistics = statistics.create_zeroed_stats_dicts()
 
     return detailed_service_schema.dump(services.values(), many=True).data
+
+
+@service_blueprint.route('/<uuid:service_id>/whitelist', methods=['GET'])
+def get_whitelist(service_id):
+    from app.models import (EMAIL_TYPE, MOBILE_TYPE)
+    service = dao_fetch_service_by_id(service_id)
+
+    if not service:
+        raise InvalidRequest("Service does not exist", status_code=404)
+
+    whitelist = dao_fetch_service_whitelist(service.id)
+    return jsonify(
+        email_addresses=[item.recipient for item in whitelist
+                         if item.recipient_type == EMAIL_TYPE],
+        phone_numbers=[item.recipient for item in whitelist
+                       if item.recipient_type == MOBILE_TYPE]
+    )
+
+
+@service_blueprint.route('/<uuid:service_id>/whitelist', methods=['PUT'])
+def update_whitelist(service_id):
+    # doesn't commit so if there are any errors, we preserve old values in db
+    dao_remove_service_whitelist(service_id)
+    try:
+        whitelist_objs = get_whitelist_objects(service_id, request.get_json())
+    except ValueError as e:
+        current_app.logger.exception(e)
+        dao_rollback()
+        msg = '{} is not a valid email address or phone number'.format(str(e))
+        return jsonify(result='error', message=msg), 400
+    else:
+        dao_add_and_commit_whitelisted_contacts(whitelist_objs)
+        return '', 204

app/service/utils.py

@@ -0,0 +1,53 @@
+import itertools
+
+from app.models import (
+    ServiceWhitelist,
+    MOBILE_TYPE, EMAIL_TYPE,
+    KEY_TYPE_TEST, KEY_TYPE_TEAM, KEY_TYPE_NORMAL)
+
+from notifications_utils.recipients import allowed_to_send_to
+
+
+def get_recipients_from_request(request_json, key, type):
+    return [(type, recipient) for recipient in request_json.get(key)]
+
+
+def get_whitelist_objects(service_id, request_json):
+    return [
+        ServiceWhitelist.from_string(service_id, type, recipient)
+        for type, recipient in (
+            get_recipients_from_request(request_json,
+                                        'phone_numbers',
+                                        MOBILE_TYPE) +
+            get_recipients_from_request(request_json,
+                                        'email_addresses',
+                                        EMAIL_TYPE)
+        )
+    ]
+
+
+def service_allowed_to_send_to(recipient, service, key_type):
+    if key_type == KEY_TYPE_TEST:
+        return True
+
+    if key_type == KEY_TYPE_NORMAL and not service.restricted:
+        return True
+
+    team_members = itertools.chain.from_iterable(
+        [user.mobile_number, user.email_address] for user in service.users)
+
+    if key_type == KEY_TYPE_TEAM:
+        return allowed_to_send_to(
+            recipient,
+            team_members
+        )
+
+    if key_type == KEY_TYPE_NORMAL and service.restricted:
+        whitelist_members = [member.recipient for member in service.whitelist]
+        return allowed_to_send_to(
+            recipient,
+            itertools.chain(
+                team_members,
+                whitelist_members
+            )
+        )

migrations/versions/0055_service_whitelist.py

@@ -0,0 +1,31 @@
+"""add service whitelist table
+
+Revision ID: 0055_service_whitelist
+Revises: 0054_perform_drop_status_column
+Create Date: 2016-09-20 12:12:30.838095
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0055_service_whitelist'
+down_revision = '0054_perform_drop_status_column'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+def upgrade():
+    op.create_table('service_whitelist',
+        sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False),
+        sa.Column('recipient_type', sa.Enum('mobile', 'email', name='recipient_type'), nullable=False),
+        sa.Column('recipient', sa.String(length=255), nullable=True),
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.ForeignKeyConstraint(['service_id'], ['services.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_service_whitelist_service_id'), 'service_whitelist', ['service_id'], unique=False)
+
+
+def downgrade():
+    op.drop_table('service_whitelist')