From 08c1ad75c8c520e39d4e8ede6622064ea97f7143 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 10 May 2023 08:39:50 -0700 Subject: [PATCH 1/6] notify-260 remove server-side timezone handling --- app/celery/nightly_tasks.py | 17 ++++----- app/celery/reporting_tasks.py | 5 +-- .../performance_platform_client.py | 3 +- app/commands.py | 6 +-- app/dao/complaint_dao.py | 6 +-- app/dao/date_util.py | 26 +++---------- app/dao/fact_billing_dao.py | 13 +++---- app/dao/fact_notification_status_dao.py | 18 ++++----- app/dao/notifications_dao.py | 6 +-- app/dao/provider_details_dao.py | 5 +-- app/dao/services_dao.py | 9 ++--- app/models.py | 4 +- app/performance_platform/processing_time.py | 6 +-- app/platform_stats/rest.py | 6 +-- app/service/rest.py | 5 +-- app/service/statistics.py | 4 +- app/utils.py | 11 +++--- tests/app/celery/test_nightly_tasks.py | 22 +++++------ tests/app/celery/test_reporting_tasks.py | 6 +-- .../notification_dao/test_notification_dao.py | 2 +- tests/app/dao/test_complaint_dao.py | 2 +- tests/app/dao/test_date_utils.py | 23 ++++++----- tests/app/dao/test_fact_billing_dao.py | 29 +++++++------- tests/app/dao/test_inbound_sms_dao.py | 22 +++++------ tests/app/dao/test_jobs_dao.py | 4 +- tests/app/dao/test_services_dao.py | 4 +- tests/app/inbound_sms/test_rest.py | 4 +- tests/app/job/test_rest.py | 4 +- tests/app/service/test_utils.py | 2 +- tests/app/test_model.py | 6 +-- tests/app/test_utils.py | 38 +++++++++---------- 31 files changed, 143 insertions(+), 175 deletions(-) diff --git a/app/celery/nightly_tasks.py b/app/celery/nightly_tasks.py index ce98dd27c..07a9fd31b 100644 --- a/app/celery/nightly_tasks.py +++ b/app/celery/nightly_tasks.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta from flask import current_app -from notifications_utils.timezones import convert_utc_to_local_timezone from sqlalchemy.exc import SQLAlchemyError from app import notify_celery @@ -25,7 +24,7 @@ fetch_service_data_retention_for_all_services_by_notification_type, ) from app.models import EMAIL_TYPE, SMS_TYPE, FactProcessingTime -from app.utils import get_local_midnight_in_utc +from app.utils import get_midnight_in_utc @notify_celery.task(name="remove_sms_email_jobs") @@ -64,9 +63,8 @@ def _delete_notifications_older_than_retention_by_type(notification_type): flexible_data_retention = fetch_service_data_retention_for_all_services_by_notification_type(notification_type) for f in flexible_data_retention: - day_to_delete_backwards_from = get_local_midnight_in_utc( - convert_utc_to_local_timezone(datetime.utcnow()).date() - timedelta(days=f.days_of_retention) - ) + day_to_delete_backwards_from = get_midnight_in_utc(datetime.utcnow()).date() \ + - timedelta(days=f.days_of_retention) delete_notifications_for_service_and_type.apply_async(queue=QueueNames.REPORTING, kwargs={ 'service_id': f.service_id, @@ -74,9 +72,8 @@ def _delete_notifications_older_than_retention_by_type(notification_type): 'datetime_to_delete_before': day_to_delete_backwards_from }) - seven_days_ago = get_local_midnight_in_utc( - convert_utc_to_local_timezone(datetime.utcnow()).date() - timedelta(days=7) - ) + seven_days_ago = get_midnight_in_utc(datetime.utcnow()).date() - timedelta(days=7) + service_ids_with_data_retention = {x.service_id for x in flexible_data_retention} # get a list of all service ids that we'll need to delete for. Typically that might only be 5% of services. @@ -168,8 +165,8 @@ def save_daily_notification_processing_time(local_date=None): else: local_date = datetime.strptime(local_date, "%Y-%m-%d").date() - start_time = get_local_midnight_in_utc(local_date) - end_time = get_local_midnight_in_utc(local_date + timedelta(days=1)) + start_time = get_midnight_in_utc(local_date) + end_time = get_midnight_in_utc(local_date + timedelta(days=1)) result = dao_get_notifications_processing_time_stats(start_time, end_time) insert_update_processing_time( FactProcessingTime( diff --git a/app/celery/reporting_tasks.py b/app/celery/reporting_tasks.py index 123eb8ced..5229f7ec6 100644 --- a/app/celery/reporting_tasks.py +++ b/app/celery/reporting_tasks.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta from flask import current_app -from notifications_utils.timezones import convert_utc_to_local_timezone from app import notify_celery from app.config import QueueNames @@ -21,7 +20,7 @@ def create_nightly_billing(day_start=None): # day_start is a datetime.date() object. e.g. # up to 4 days of data counting back from day_start is consolidated if day_start is None: - day_start = convert_utc_to_local_timezone(datetime.utcnow()).date() - timedelta(days=1) + day_start = datetime.utcnow().date() - timedelta(days=1) else: # When calling the task its a string in the format of "YYYY-MM-DD" day_start = datetime.strptime(day_start, "%Y-%m-%d").date() @@ -83,7 +82,7 @@ def create_nightly_notification_status(): mean the aggregated results are temporarily incorrect. """ - yesterday = convert_utc_to_local_timezone(datetime.utcnow()).date() - timedelta(days=1) + yesterday = datetime.utcnow().date() - timedelta(days=1) for notification_type in [SMS_TYPE, EMAIL_TYPE]: days = 4 diff --git a/app/clients/performance_platform/performance_platform_client.py b/app/clients/performance_platform/performance_platform_client.py index 6a27b402f..6448fca75 100644 --- a/app/clients/performance_platform/performance_platform_client.py +++ b/app/clients/performance_platform/performance_platform_client.py @@ -3,7 +3,6 @@ import requests from flask import current_app -from notifications_utils.timezones import convert_utc_to_local_timezone class PerformancePlatformClient: @@ -55,7 +54,7 @@ def format_payload(*, dataset, start_time, group_name, group_value, count, perio :param period - the period that this data covers - "day", "week", "month", "quarter". """ payload = { - '_timestamp': convert_utc_to_local_timezone(start_time).isoformat(), + '_timestamp': start_time, 'service': 'govuk-notify', 'dataType': dataset, 'period': period, diff --git a/app/commands.py b/app/commands.py index 7a8c99d7b..e0f676b08 100644 --- a/app/commands.py +++ b/app/commands.py @@ -62,7 +62,7 @@ TemplateHistory, User, ) -from app.utils import get_local_midnight_in_utc +from app.utils import get_midnight_in_utc @click.group(name='command', help='Additional commands') @@ -192,8 +192,8 @@ def rebuild_ft_data(process_day, service): rebuild_ft_data(day, service_id) else: services = get_service_ids_that_need_billing_populated( - get_local_midnight_in_utc(day), - get_local_midnight_in_utc(day + timedelta(days=1)) + get_midnight_in_utc(day), + get_midnight_in_utc(day + timedelta(days=1)) ) for row in services: rebuild_ft_data(day, row.service_id) diff --git a/app/dao/complaint_dao.py b/app/dao/complaint_dao.py index 819a3666f..9a9351912 100644 --- a/app/dao/complaint_dao.py +++ b/app/dao/complaint_dao.py @@ -6,7 +6,7 @@ from app import db from app.dao.dao_utils import autocommit from app.models import Complaint -from app.utils import get_local_midnight_in_utc +from app.utils import get_midnight_in_utc @autocommit @@ -28,7 +28,7 @@ def fetch_complaints_by_service(service_id): def fetch_count_of_complaints(start_date, end_date): - start_date = get_local_midnight_in_utc(start_date) - end_date = get_local_midnight_in_utc(end_date + timedelta(days=1)) + start_date = get_midnight_in_utc(start_date) + end_date = get_midnight_in_utc(end_date + timedelta(days=1)) return Complaint.query.filter(Complaint.created_at >= start_date, Complaint.created_at < end_date).count() diff --git a/app/dao/date_util.py b/app/dao/date_util.py index 94d6cb3cb..980661cf7 100644 --- a/app/dao/date_util.py +++ b/app/dao/date_util.py @@ -1,20 +1,13 @@ from datetime import date, datetime, time, timedelta -import pytz -from notifications_utils.timezones import ( - convert_local_timezone_to_utc, - convert_utc_to_local_timezone, - local_timezone, -) - def get_months_for_financial_year(year): return [ - convert_local_timezone_to_utc(month) for month in ( + month for month in ( get_months_for_year(4, 13, year) + get_months_for_year(1, 4, year + 1) ) - if convert_local_timezone_to_utc(month) < datetime.now() + if month < datetime.now() ] @@ -30,8 +23,8 @@ def get_financial_year_dates(year): year_start_datetime, year_end_datetime = get_financial_year(year) return ( - convert_utc_to_local_timezone(year_start_datetime).date(), - convert_utc_to_local_timezone(year_end_datetime).date() + year_start_datetime.date(), + year_end_datetime.date() ) @@ -44,14 +37,7 @@ def get_current_financial_year(): def get_april_fools(year): - """ - This function converts the start of the financial year April 1, 00:00 as BST (British Standard Time) to UTC, - the tzinfo is lastly removed from the datetime because the database stores the timestamps without timezone. - :param year: the year to calculate the April 1, 00:00 BST for - :return: the datetime of April 1 for the given year, for example 2016 = 2016-03-31 23:00:00 - """ - return local_timezone.localize( - datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace(tzinfo=None) + return datetime(year, 4, 1, 0, 0, 0) def get_month_start_and_end_date_in_utc(month_year): @@ -64,7 +50,7 @@ def get_month_start_and_end_date_in_utc(month_year): _, num_days = calendar.monthrange(month_year.year, month_year.month) first_day = datetime(month_year.year, month_year.month, 1, 0, 0, 0) last_day = datetime(month_year.year, month_year.month, num_days, 23, 59, 59, 99999) - return convert_local_timezone_to_utc(first_day), convert_local_timezone_to_utc(last_day) + return first_day, last_day def get_current_financial_year_start_year(): diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py index 187b2c860..7cb5d97e5 100644 --- a/app/dao/fact_billing_dao.py +++ b/app/dao/fact_billing_dao.py @@ -1,7 +1,6 @@ from datetime import date, datetime, timedelta from flask import current_app -from notifications_utils.timezones import convert_utc_to_local_timezone from sqlalchemy import Date, Integer, and_, desc, func, union from sqlalchemy.dialects.postgresql import insert from sqlalchemy.sql.expression import case, literal @@ -27,7 +26,7 @@ Rate, Service, ) -from app.utils import get_local_midnight_in_utc +from app.utils import get_midnight_in_utc def fetch_sms_free_allowance_remainder_until_date(end_date): @@ -179,7 +178,7 @@ def fetch_monthly_billing_for_year(service_id, year): we also update the table on-the-fly if we need accurate data for this year. """ _, year_end = get_financial_year_dates(year) - today = convert_utc_to_local_timezone(datetime.utcnow()).date() + today = datetime.utcnow().date() # if year end date is less than today, we are calculating for data in the past and have no need for deltas. if year_end >= today: @@ -330,8 +329,8 @@ def delete_billing_data_for_service_for_day(process_day, service_id): def fetch_billing_data_for_day(process_day, service_id=None, check_permissions=False): - start_date = get_local_midnight_in_utc(process_day) - end_date = get_local_midnight_in_utc(process_day + timedelta(days=1)) + start_date = get_midnight_in_utc(process_day) + end_date = get_midnight_in_utc(process_day + timedelta(days=1)) current_app.logger.info("Populate ft_billing for {} to {}".format(start_date, end_date)) transit_data = [] if not service_id: @@ -430,7 +429,7 @@ def get_service_ids_that_need_billing_populated(start_date, end_date): def get_rate( rates, notification_type, date ): - start_of_day = get_local_midnight_in_utc(date) + start_of_day = get_midnight_in_utc(date) if notification_type == SMS_TYPE: return next( @@ -624,7 +623,7 @@ def query_organisation_sms_usage_for_year(organisation_id, year): def fetch_usage_year_for_organisation(organisation_id, year): year_start, year_end = get_financial_year_dates(year) - today = convert_utc_to_local_timezone(datetime.utcnow()).date() + today = datetime.utcnow().date() services = dao_get_organisation_live_services(organisation_id) # if year end date is less than today, we are calculating for data in the past and have no need for deltas. diff --git a/app/dao/fact_notification_status_dao.py b/app/dao/fact_notification_status_dao.py index e4ab55203..aa884cc2d 100644 --- a/app/dao/fact_notification_status_dao.py +++ b/app/dao/fact_notification_status_dao.py @@ -28,16 +28,16 @@ Template, ) from app.utils import ( - get_local_midnight_in_utc, get_local_month_from_utc_column, + get_midnight_in_utc, midnight_n_days_ago, ) @autocommit def update_fact_notification_status(process_day, notification_type, service_id): - start_date = get_local_midnight_in_utc(process_day) - end_date = get_local_midnight_in_utc(process_day + timedelta(days=1)) + start_date = get_midnight_in_utc(process_day) + end_date = get_midnight_in_utc(process_day + timedelta(days=1)) # delete any existing rows in case some no longer exist e.g. if all messages are sent FactNotificationStatus.query.filter( @@ -112,8 +112,8 @@ def fetch_notification_status_for_service_for_day(bst_day, service_id): Notification.status.label('notification_status'), func.count().label('count') ).filter( - Notification.created_at >= get_local_midnight_in_utc(bst_day), - Notification.created_at < get_local_midnight_in_utc(bst_day + timedelta(days=1)), + Notification.created_at >= get_midnight_in_utc(bst_day), + Notification.created_at < get_midnight_in_utc(bst_day + timedelta(days=1)), Notification.service_id == service_id, Notification.key_type != KEY_TYPE_TEST ).group_by( @@ -142,7 +142,7 @@ def fetch_notification_status_for_service_for_today_and_7_previous_days(service_ *([Notification.template_id] if by_template else []), func.count().label('count') ).filter( - Notification.created_at >= get_local_midnight_in_utc(now), + Notification.created_at >= get_midnight_in_utc(now), Notification.service_id == service_id, Notification.key_type != KEY_TYPE_TEST ).group_by( @@ -188,7 +188,7 @@ def fetch_notification_status_totals_for_all_services(start_date, end_date): FactNotificationStatus.notification_status, FactNotificationStatus.key_type, ) - today = get_local_midnight_in_utc(datetime.utcnow()) + today = get_midnight_in_utc(datetime.utcnow()) if start_date <= datetime.utcnow().date() <= end_date: stats_for_today = db.session.query( Notification.notification_type.cast(db.Text).label('notification_type'), @@ -265,7 +265,7 @@ def fetch_stats_for_all_services_by_date_range(start_date, end_date, include_fro stats = stats.filter(FactNotificationStatus.key_type != KEY_TYPE_TEST) if start_date <= datetime.utcnow().date() <= end_date: - today = get_local_midnight_in_utc(datetime.utcnow()) + today = get_midnight_in_utc(datetime.utcnow()) subquery = db.session.query( Notification.notification_type.cast(db.Text).label('notification_type'), Notification.status.label('status'), @@ -357,7 +357,7 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id): ) if start_date <= datetime.utcnow() <= end_date: - today = get_local_midnight_in_utc(datetime.utcnow()) + today = get_midnight_in_utc(datetime.utcnow()) month = get_local_month_from_utc_column(Notification.created_at) stats_for_today = db.session.query( diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index d560e61eb..72a96d22b 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -35,7 +35,7 @@ ) from app.utils import ( escape_special_characters, - get_local_midnight_in_utc, + get_midnight_in_utc, midnight_n_days_ago, ) @@ -579,8 +579,8 @@ def get_service_ids_with_notifications_before(notification_type, timestamp): def get_service_ids_with_notifications_on_date(notification_type, date): - start_date = get_local_midnight_in_utc(date) - end_date = get_local_midnight_in_utc(date + timedelta(days=1)) + start_date = get_midnight_in_utc(date) + end_date = get_midnight_in_utc(date + timedelta(days=1)) notification_table_query = db.session.query( Notification.service_id.label('service_id') diff --git a/app/dao/provider_details_dao.py b/app/dao/provider_details_dao.py index 2fc03d424..e32e26e1e 100644 --- a/app/dao/provider_details_dao.py +++ b/app/dao/provider_details_dao.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta from flask import current_app -from notifications_utils.timezones import convert_utc_to_local_timezone from sqlalchemy import asc, desc, func from app import db @@ -155,8 +154,8 @@ def _update_provider_details_without_commit(provider_details): def dao_get_provider_stats(): # this query does not include the current day since the task to populate ft_billing runs overnight - current_local_datetime = convert_utc_to_local_timezone(datetime.utcnow()) - first_day_of_the_month = current_local_datetime.date().replace(day=1) + current_datetime = datetime.utcnow() + first_day_of_the_month = current_datetime.date().replace(day=1) subquery = db.session.query( FactBilling.provider, diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 33caf9195..4456d32b2 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -42,7 +42,7 @@ from app.utils import ( escape_special_characters, get_archived_db_column_value, - get_local_midnight_in_utc, + get_midnight_in_utc, ) DEFAULT_SERVICE_PERMISSIONS = [ @@ -388,8 +388,7 @@ def _delete_commit(query): def dao_fetch_todays_stats_for_service(service_id): today = date.today() - start_date = get_local_midnight_in_utc(today) - + start_date = get_midnight_in_utc(today) return db.session.query( Notification.notification_type, Notification.status, @@ -406,8 +405,8 @@ def dao_fetch_todays_stats_for_service(service_id): def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_active=True): today = date.today() - start_date = get_local_midnight_in_utc(today) - end_date = get_local_midnight_in_utc(today + timedelta(days=1)) + start_date = get_midnight_in_utc(today) + end_date = get_midnight_in_utc(today + timedelta(days=1)) subquery = db.session.query( Notification.notification_type, diff --git a/app/models.py b/app/models.py index 31741b41a..50d2bd86b 100644 --- a/app/models.py +++ b/app/models.py @@ -17,7 +17,6 @@ PlainTextEmailTemplate, SMSMessageTemplate, ) -from notifications_utils.timezones import convert_utc_to_local_timezone from sqlalchemy import CheckConstraint, Index, UniqueConstraint from sqlalchemy.dialects.postgresql import JSON, JSONB, UUID from sqlalchemy.ext.associationproxy import association_proxy @@ -1503,7 +1502,6 @@ def get_created_by_email_address(self): return None def serialize_for_csv(self): - created_at_in_est = convert_utc_to_local_timezone(self.created_at) serialized = { "row_number": '' if self.job_row_number is None else self.job_row_number + 1, "recipient": self.to, @@ -1512,7 +1510,7 @@ def serialize_for_csv(self): "template_type": self.template.template_type, "job_name": self.job.original_file_name if self.job else '', "status": self.formatted_status, - "created_at": created_at_in_est.strftime("%Y-%m-%d %H:%M:%S"), + "created_at": self.created_at.strftime("%Y-%m-%d %H:%M:%S"), "created_by_name": self.get_created_by_name(), "created_by_email_address": self.get_created_by_email_address(), } diff --git a/app/performance_platform/processing_time.py b/app/performance_platform/processing_time.py index f58c94d43..25e16c5b6 100644 --- a/app/performance_platform/processing_time.py +++ b/app/performance_platform/processing_time.py @@ -6,12 +6,12 @@ from app.dao.notifications_dao import ( dao_get_total_notifications_sent_per_day_for_performance_platform, ) -from app.utils import get_local_midnight_in_utc +from app.utils import get_midnight_in_utc def send_processing_time_to_performance_platform(local_date): - start_time = get_local_midnight_in_utc(local_date) - end_time = get_local_midnight_in_utc(local_date + timedelta(days=1)) + start_time = get_midnight_in_utc(local_date) + end_time = get_midnight_in_utc(local_date + timedelta(days=1)) send_processing_time_for_start_and_end(start_time, end_time, local_date) diff --git a/app/platform_stats/rest.py b/app/platform_stats/rest.py index 554fc3988..fa340450b 100644 --- a/app/platform_stats/rest.py +++ b/app/platform_stats/rest.py @@ -17,7 +17,7 @@ from app.platform_stats.platform_stats_schema import platform_stats_request from app.schema_validation import validate from app.service.statistics import format_admin_stats -from app.utils import get_local_midnight_in_utc +from app.utils import get_midnight_in_utc platform_stats_blueprint = Blueprint('platform_stats', __name__) @@ -54,8 +54,8 @@ def validate_date_range_is_within_a_financial_year(start_date, end_date): if end_date < start_date: raise InvalidRequest(message="Start date must be before end date", status_code=400) - start_fy = get_financial_year_for_datetime(get_local_midnight_in_utc(start_date)) - end_fy = get_financial_year_for_datetime(get_local_midnight_in_utc(end_date)) + start_fy = get_financial_year_for_datetime(get_midnight_in_utc(start_date)) + end_fy = get_financial_year_for_datetime(get_midnight_in_utc(end_date)) if start_fy != end_fy: raise InvalidRequest(message="Date must be in a single financial year.", status_code=400) diff --git a/app/service/rest.py b/app/service/rest.py index d969f4f5a..abeb41e48 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -2,7 +2,6 @@ from datetime import datetime from flask import Blueprint, current_app, jsonify, request -from notifications_utils.timezones import convert_utc_to_local_timezone from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import NoResultFound from werkzeug.datastructures import MultiDict @@ -500,9 +499,7 @@ def get_monthly_notification_stats(service_id): now = datetime.utcnow() if end_date > now: - todays_deltas = fetch_notification_status_for_service_for_day( - convert_utc_to_local_timezone(now), service_id=service_id - ) + todays_deltas = fetch_notification_status_for_service_for_day(now, service_id=service_id) statistics.add_monthly_notification_status_stats(data, todays_deltas) return jsonify(data=data) diff --git a/app/service/statistics.py b/app/service/statistics.py index e36106f0b..4a45189ad 100644 --- a/app/service/statistics.py +++ b/app/service/statistics.py @@ -1,8 +1,6 @@ from collections import defaultdict from datetime import datetime -from notifications_utils.timezones import convert_utc_to_local_timezone - from app.dao.date_util import get_months_for_financial_year from app.models import NOTIFICATION_STATUS_TYPES, NOTIFICATION_TYPES @@ -98,7 +96,7 @@ def create_empty_monthly_notification_status_stats_dict(year): utc_month_starts = get_months_for_financial_year(year) # nested dicts - data[month][template type][status] = count return { - convert_utc_to_local_timezone(start).strftime('%Y-%m'): { + start.strftime('%Y-%m'): { template_type: defaultdict(int) for template_type in NOTIFICATION_TYPES } diff --git a/app/utils.py b/app/utils.py index dbd9b3056..622582997 100644 --- a/app/utils.py +++ b/app/utils.py @@ -3,7 +3,6 @@ from flask import url_for from notifications_utils.template import HTMLEmailTemplate, SMSMessageTemplate -from notifications_utils.timezones import convert_local_timezone_to_utc from sqlalchemy import func DATETIME_FORMAT_NO_TIMEZONE = "%Y-%m-%d %H:%M:%S.%f" @@ -49,19 +48,19 @@ def get_template_instance(template, values): }[template['template_type']](template, values) -def get_local_midnight_in_utc(date): +def get_midnight_in_utc(date): """ - This function converts date from midnight in local time to UTC, + This function converts date to midnight in UTC, removing the tzinfo from the datetime because the database stores the timestamps without timezone. :param date: the day to calculate the local midnight in UTC for :return: the datetime of local midnight in UTC, for example 2016-06-17 = 2016-06-16 23:00:00 """ - return convert_local_timezone_to_utc(datetime.combine(date, datetime.min.time())) + return datetime.combine(date, datetime.min.time()) def get_midnight_for_day_before(date): day_before = date - timedelta(1) - return get_local_midnight_in_utc(day_before) + return get_midnight_in_utc(day_before) def get_local_month_from_utc_column(column): @@ -95,7 +94,7 @@ def midnight_n_days_ago(number_of_days): """ Returns midnight a number of days ago. Takes care of daylight savings etc. """ - return get_local_midnight_in_utc(datetime.utcnow() - timedelta(days=number_of_days)) + return get_midnight_in_utc(datetime.utcnow() - timedelta(days=number_of_days)) def escape_special_characters(string): diff --git a/tests/app/celery/test_nightly_tasks.py b/tests/app/celery/test_nightly_tasks.py index 0599799e4..8c4d50b9e 100644 --- a/tests/app/celery/test_nightly_tasks.py +++ b/tests/app/celery/test_nightly_tasks.py @@ -239,11 +239,11 @@ def test_delete_notifications_task_calls_task_for_services_with_data_retention_o 'service_id': sms_service.id, 'notification_type': 'sms', # three days of retention, its morn of 5th, so we want to keep all messages from 4th, 3rd and 2nd. - 'datetime_to_delete_before': datetime(2021, 6, 2, 4, 0), + 'datetime_to_delete_before': date(2021, 6, 2), }) -@freeze_time('2021-04-05 03:00') +@freeze_time('2021-04-04 23:00') def test_delete_notifications_task_calls_task_for_services_with_data_retention_by_looking_at_retention( notify_db_session, mocker @@ -262,17 +262,17 @@ def test_delete_notifications_task_calls_task_for_services_with_data_retention_b call(queue=ANY, kwargs={ 'service_id': service_14_days.id, 'notification_type': 'sms', - 'datetime_to_delete_before': datetime(2021, 3, 21, 4, 0) + 'datetime_to_delete_before': date(2021, 3, 21) }), call(queue=ANY, kwargs={ 'service_id': service_3_days.id, 'notification_type': 'sms', - 'datetime_to_delete_before': datetime(2021, 4, 1, 4, 0) + 'datetime_to_delete_before': date(2021, 4, 1) }), ]) -@freeze_time('2021-04-03 03:00') +@freeze_time('2021-04-02 23:00') def test_delete_notifications_task_calls_task_for_services_that_have_sent_notifications_recently( notify_db_session, mocker @@ -288,13 +288,13 @@ def test_delete_notifications_task_calls_task_for_services_that_have_sent_notifi nothing_to_delete_email_template = create_template(service_nothing_to_delete, template_type='email') # will be deleted as service has no custom retention, but past our default 7 days - create_notification(service_will_delete_1.templates[0], created_at=datetime.now() - timedelta(days=8)) - create_notification(service_will_delete_2.templates[0], created_at=datetime.now() - timedelta(days=8)) + create_notification(service_will_delete_1.templates[0], created_at=datetime.utcnow() - timedelta(days=8)) + create_notification(service_will_delete_2.templates[0], created_at=datetime.utcnow() - timedelta(days=8)) # will be kept as it's recent, and we won't run delete_notifications_for_service_and_type - create_notification(nothing_to_delete_sms_template, created_at=datetime.now() - timedelta(days=2)) + create_notification(nothing_to_delete_sms_template, created_at=datetime.utcnow() - timedelta(days=2)) # this is an old notification, but for email not sms, so we won't run delete_notifications_for_service_and_type - create_notification(nothing_to_delete_email_template, created_at=datetime.now() - timedelta(days=8)) + create_notification(nothing_to_delete_email_template, created_at=datetime.utcnow() - timedelta(days=8)) mock_subtask = mocker.patch('app.celery.nightly_tasks.delete_notifications_for_service_and_type') @@ -305,11 +305,11 @@ def test_delete_notifications_task_calls_task_for_services_that_have_sent_notifi call(queue=ANY, kwargs={ 'service_id': service_will_delete_1.id, 'notification_type': 'sms', - 'datetime_to_delete_before': datetime(2021, 3, 26, 4, 0) + 'datetime_to_delete_before': date(2021, 3, 26) }), call(queue=ANY, kwargs={ 'service_id': service_will_delete_2.id, 'notification_type': 'sms', - 'datetime_to_delete_before': datetime(2021, 3, 26, 4, 0) + 'datetime_to_delete_before': date(2021, 3, 26) }), ]) diff --git a/tests/app/celery/test_reporting_tasks.py b/tests/app/celery/test_reporting_tasks.py index 987e4bc8f..fb50c9deb 100644 --- a/tests/app/celery/test_reporting_tasks.py +++ b/tests/app/celery/test_reporting_tasks.py @@ -73,7 +73,7 @@ def test_create_nightly_notification_status_triggers_tasks( mock_celery.assert_called_with( kwargs={ 'service_id': sample_service.id, - 'process_day': '2019-07-30', + 'process_day': '2019-07-31', 'notification_type': SMS_TYPE }, queue=QueueNames.REPORTING @@ -352,7 +352,7 @@ def test_create_nightly_billing_for_day_use_BST( ) create_notification( - created_at=datetime(2018, 3, 26, 3, 59), + created_at=datetime(2018, 3, 25, 23, 59), template=sample_template, status='delivered', rate_multiplier=1.0, @@ -361,7 +361,7 @@ def test_create_nightly_billing_for_day_use_BST( # too early create_notification( - created_at=datetime(2018, 3, 25, 3, 59), + created_at=datetime(2018, 3, 24, 23, 59), template=sample_template, status='delivered', rate_multiplier=1.0, diff --git a/tests/app/dao/notification_dao/test_notification_dao.py b/tests/app/dao/notification_dao/test_notification_dao.py index 34aac0466..5eecdd087 100644 --- a/tests/app/dao/notification_dao/test_notification_dao.py +++ b/tests/app/dao/notification_dao/test_notification_dao.py @@ -1495,7 +1495,7 @@ def test_notifications_not_yet_sent_return_no_rows(sample_service, notification_ @pytest.mark.parametrize('created_at_utc,date_to_check,expected_count', [ # Clocks change on the 27th of March 2022, so the query needs to look at the # time range 00:00 - 23:00 (UTC) thereafter. - ('2022-03-27T00:30', date(2022, 3, 27), 0), # 27/03 00:30 GMT + ('2022-03-27T00:30', date(2022, 3, 27), 1), # 27/03 00:30 GMT ('2022-03-27T22:30', date(2022, 3, 27), 1), # 27/03 23:30 BST ('2022-03-27T23:30', date(2022, 3, 27), 1), # 28/03 00:30 BST ('2022-03-26T23:30', date(2022, 3, 26), 1), # 26/03 23:30 GMT diff --git a/tests/app/dao/test_complaint_dao.py b/tests/app/dao/test_complaint_dao.py index 290337fb9..af16b8af2 100644 --- a/tests/app/dao/test_complaint_dao.py +++ b/tests/app/dao/test_complaint_dao.py @@ -117,7 +117,7 @@ def test_fetch_count_of_complaints(sample_email_notification): count_of_complaints = fetch_count_of_complaints(start_date=datetime(2018, 6, 7), end_date=datetime(2018, 6, 7)) - assert count_of_complaints == 3 + assert count_of_complaints == 5 def test_fetch_count_of_complaints_returns_zero(notify_db_session): diff --git a/tests/app/dao/test_date_utils.py b/tests/app/dao/test_date_utils.py index f153c64d0..c09e18968 100644 --- a/tests/app/dao/test_date_utils.py +++ b/tests/app/dao/test_date_utils.py @@ -12,22 +12,22 @@ def test_get_financial_year(): start, end = get_financial_year(2000) - assert str(start) == '2000-04-01 05:00:00' - assert str(end) == '2001-04-01 04:59:59.999999' + assert str(start) == '2000-04-01 00:00:00' + assert str(end) == '2001-03-31 23:59:59.999999' def test_get_april_fools(): april_fools = get_april_fools(2016) - assert str(april_fools) == '2016-04-01 04:00:00' + assert str(april_fools) == '2016-04-01 00:00:00' assert april_fools.tzinfo is None @pytest.mark.parametrize("month, year, expected_start, expected_end", [ - (7, 2017, datetime(2017, 7, 1, 4, 00, 00), datetime(2017, 8, 1, 3, 59, 59, 99999)), - (2, 2016, datetime(2016, 2, 1, 5, 00, 00), datetime(2016, 3, 1, 4, 59, 59, 99999)), - (2, 2017, datetime(2017, 2, 1, 5, 00, 00), datetime(2017, 3, 1, 4, 59, 59, 99999)), - (9, 2018, datetime(2018, 9, 1, 4, 00, 00), datetime(2018, 10, 1, 3, 59, 59, 99999)), - (12, 2019, datetime(2019, 12, 1, 5, 00, 00), datetime(2020, 1, 1, 4, 59, 59, 99999)) + (7, 2017, datetime(2017, 7, 1, 0, 00, 00), datetime(2017, 7, 31, 23, 59, 59, 99999)), + (2, 2016, datetime(2016, 2, 1, 0, 00, 00), datetime(2016, 2, 29, 23, 59, 59, 99999)), + (2, 2017, datetime(2017, 2, 1, 0, 00, 00), datetime(2017, 2, 28, 23, 59, 59, 99999)), + (9, 2018, datetime(2018, 9, 1, 0, 00, 00), datetime(2018, 9, 30, 23, 59, 59, 99999)), + (12, 2019, datetime(2019, 12, 1, 0, 00, 00), datetime(2019, 12, 31, 23, 59, 59, 99999)) ]) def test_get_month_start_and_end_date_in_utc(month, year, expected_start, expected_end): month_year = datetime(year, month, 10, 13, 30, 00) @@ -37,11 +37,10 @@ def test_get_month_start_and_end_date_in_utc(month, year, expected_start, expect @pytest.mark.parametrize("dt, fy", [ - (datetime(2018, 4, 1, 4, 0, 0), 2018), - (datetime(2019, 4, 1, 3, 59, 59), 2018), - (datetime(2019, 4, 1, 4, 0, 0), 2019), + (datetime(2018, 4, 1, 1, 0, 0), 2018), + (datetime(2019, 3, 31, 23, 59, 59), 2018), (date(2019, 3, 31), 2018), - (date(2019, 4, 2), 2019), # date() gives midnight UTC, which is the day before in ET + (date(2019, 4, 2), 2019), ]) def test_get_financial_year_for_datetime(dt, fy): assert get_financial_year_for_datetime(dt) == fy diff --git a/tests/app/dao/test_fact_billing_dao.py b/tests/app/dao/test_fact_billing_dao.py index edb4e3692..ecc71efa5 100644 --- a/tests/app/dao/test_fact_billing_dao.py +++ b/tests/app/dao/test_fact_billing_dao.py @@ -3,7 +3,6 @@ import pytest from freezegun import freeze_time -from notifications_utils.timezones import convert_utc_to_local_timezone from app import db from app.dao.fact_billing_dao import ( @@ -74,7 +73,7 @@ def test_fetch_billing_data_for_today_includes_data_with_the_right_key_type(noti for key_type in ['normal', 'test', 'team']: create_notification(template=template, status='delivered', key_type=key_type) - today = convert_utc_to_local_timezone(datetime.utcnow()) + today = datetime.utcnow() results = fetch_billing_data_for_day(today.date()) assert len(results) == 1 assert results[0].notifications_sent == 2 @@ -87,7 +86,7 @@ def test_fetch_billing_data_for_day_only_calls_query_for_permission_type(notify_ sms_template = create_template(service=service, template_type="sms") create_notification(template=email_template, status='delivered') create_notification(template=sms_template, status='delivered') - today = convert_utc_to_local_timezone(datetime.utcnow()) + today = datetime.utcnow() results = fetch_billing_data_for_day(process_day=today.date(), check_permissions=True) assert len(results) == 1 @@ -99,7 +98,7 @@ def test_fetch_billing_data_for_day_only_calls_query_for_all_channels(notify_db_ sms_template = create_template(service=service, template_type="sms") create_notification(template=email_template, status='delivered') create_notification(template=sms_template, status='delivered') - today = convert_utc_to_local_timezone(datetime.utcnow()) + today = datetime.utcnow() results = fetch_billing_data_for_day(process_day=today.date(), check_permissions=False) assert len(results) == 2 @@ -115,10 +114,10 @@ def test_fetch_billing_data_for_today_includes_data_with_the_right_date(notify_d create_notification(template=template, status='delivered', created_at=datetime(2018, 4, 1, 0, 23, 23)) create_notification(template=template, status='sending', created_at=process_day + timedelta(days=1)) - day_under_test = convert_utc_to_local_timezone(process_day) + day_under_test = process_day results = fetch_billing_data_for_day(day_under_test.date()) assert len(results) == 1 - assert results[0].notifications_sent == 2 + assert results[0].notifications_sent == 3 def test_fetch_billing_data_for_day_is_grouped_by_template_and_notification_type(notify_db_session): @@ -128,7 +127,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_template_and_notification_type create_notification(template=email_template, status='delivered') create_notification(template=sms_template, status='delivered') - today = convert_utc_to_local_timezone(datetime.utcnow()) + today = datetime.utcnow() results = fetch_billing_data_for_day(today.date()) assert len(results) == 2 assert results[0].notifications_sent == 1 @@ -143,7 +142,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_service(notify_db_session): create_notification(template=email_template, status='delivered') create_notification(template=sms_template, status='delivered') - today = convert_utc_to_local_timezone(datetime.utcnow()) + today = datetime.utcnow() results = fetch_billing_data_for_day(today.date()) assert len(results) == 2 assert results[0].notifications_sent == 1 @@ -156,7 +155,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_provider(notify_db_session): create_notification(template=template, status='delivered', sent_by='sns') create_notification(template=template, status='delivered', sent_by='sns') - today = convert_utc_to_local_timezone(datetime.utcnow()) + today = datetime.utcnow() results = fetch_billing_data_for_day(today.date()) assert len(results) == 1 assert results[0].notifications_sent == 2 @@ -169,7 +168,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_rate_mulitplier(notify_db_sess create_notification(template=template, status='delivered', rate_multiplier=1) create_notification(template=template, status='delivered', rate_multiplier=2) - today = convert_utc_to_local_timezone(datetime.utcnow()) + today = datetime.utcnow() results = fetch_billing_data_for_day(today.date()) assert len(results) == 2 assert results[0].notifications_sent == 1 @@ -182,7 +181,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_international(notify_db_sessio create_notification(template=sms_template, status='delivered', international=True) create_notification(template=sms_template, status='delivered', international=False) - today = convert_utc_to_local_timezone(datetime.utcnow()) + today = datetime.utcnow() results = fetch_billing_data_for_day(today.date()) assert len(results) == 2 assert all(result.notifications_sent == 1 for result in results) @@ -198,7 +197,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_notification_type(notify_db_se create_notification(template=email_template, status='delivered') create_notification(template=email_template, status='delivered') - today = convert_utc_to_local_timezone(datetime.utcnow()) + today = datetime.utcnow() results = fetch_billing_data_for_day(today.date()) assert len(results) == 2 notification_types = [x.notification_type for x in results] @@ -206,7 +205,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_notification_type(notify_db_se def test_fetch_billing_data_for_day_returns_empty_list(notify_db_session): - today = convert_utc_to_local_timezone(datetime.utcnow()) + today = datetime.utcnow() results = fetch_billing_data_for_day(today.date()) assert results == [] @@ -239,7 +238,7 @@ def test_fetch_billing_data_for_day_returns_list_for_given_service(notify_db_ses create_notification(template=template, status='delivered') create_notification(template=template_2, status='delivered') - today = convert_utc_to_local_timezone(datetime.utcnow()) + today = datetime.utcnow() results = fetch_billing_data_for_day(process_day=today.date(), service_id=service.id) assert len(results) == 1 assert results[0].service_id == service.id @@ -252,7 +251,7 @@ def test_fetch_billing_data_for_day_bills_correctly_for_status(notify_db_session for status in NOTIFICATION_STATUS_TYPES: create_notification(template=sms_template, status=status) create_notification(template=email_template, status=status) - today = convert_utc_to_local_timezone(datetime.utcnow()) + today = datetime.utcnow() results = fetch_billing_data_for_day(process_day=today.date(), service_id=service.id) sms_results = [x for x in results if x.notification_type == 'sms'] diff --git a/tests/app/dao/test_inbound_sms_dao.py b/tests/app/dao/test_inbound_sms_dao.py index 9f234bcbb..dfa2d6fc5 100644 --- a/tests/app/dao/test_inbound_sms_dao.py +++ b/tests/app/dao/test_inbound_sms_dao.py @@ -64,8 +64,8 @@ def test_get_all_inbound_sms_filters_on_service(notify_db_session): def test_get_all_inbound_sms_filters_on_time(sample_service, notify_db_session): - create_inbound_sms(sample_service, created_at=datetime(2017, 8, 7, 3, 59)) # sunday evening - sms_two = create_inbound_sms(sample_service, created_at=datetime(2017, 8, 7, 4, 0)) # monday (7th) morning + create_inbound_sms(sample_service, created_at=datetime(2017, 8, 6, 23, 59)) # sunday evening + sms_two = create_inbound_sms(sample_service, created_at=datetime(2017, 8, 7, 0, 0)) # monday (7th) morning with freeze_time('2017-08-14 12:00'): res = dao_get_inbound_sms_for_service(sample_service.id, limit_days=7) @@ -87,9 +87,9 @@ def test_count_inbound_sms_for_service(notify_db_session): def test_count_inbound_sms_for_service_filters_messages_older_than_n_days(sample_service): # test between evening sunday 2nd of june and morning of monday 3rd - create_inbound_sms(sample_service, created_at=datetime(2019, 6, 3, 3, 59)) - create_inbound_sms(sample_service, created_at=datetime(2019, 6, 3, 3, 59)) - create_inbound_sms(sample_service, created_at=datetime(2019, 6, 3, 4, 1)) + create_inbound_sms(sample_service, created_at=datetime(2019, 6, 2, 23, 59)) + create_inbound_sms(sample_service, created_at=datetime(2019, 6, 2, 23, 59)) + create_inbound_sms(sample_service, created_at=datetime(2019, 6, 3, 0, 1)) with freeze_time('Monday 10th June 2019 12:00'): assert dao_count_inbound_sms_for_service(sample_service.id, limit_days=7) == 1 @@ -108,10 +108,10 @@ def test_should_delete_inbound_sms_according_to_data_retention(notify_db_session create_service_data_retention(short_retention_service, notification_type='email', days_of_retention=4) dates = [ - datetime(2017, 6, 5, 4, 00), # just before three days - datetime(2017, 6, 5, 3, 59), # older than three days - datetime(2017, 6, 1, 4, 00), # just before seven days - datetime(2017, 6, 1, 3, 59), # older than seven days + datetime(2017, 6, 5, 0, 00), # just before three days + datetime(2017, 6, 4, 23, 59), # older than three days + datetime(2017, 6, 1, 0, 00), # just before seven days + datetime(2017, 5, 31, 23, 59), # older than seven days datetime(2017, 5, 1, 0, 0), # older than thirty days ] @@ -340,9 +340,9 @@ def test_most_recent_inbound_sms_paginates_properly(notify_api, sample_service): def test_most_recent_inbound_sms_only_returns_values_within_7_days(sample_service): # just out of bounds - create_inbound_sms(sample_service, user_number='1', content='old', created_at=datetime(2017, 4, 3, 3, 59, 59)) + create_inbound_sms(sample_service, user_number='1', content='old', created_at=datetime(2017, 4, 2, 23, 59, 59)) # just in bounds - create_inbound_sms(sample_service, user_number='2', content='new', created_at=datetime(2017, 4, 3, 4, 0, 0)) + create_inbound_sms(sample_service, user_number='2', content='new', created_at=datetime(2017, 4, 3, 0, 0, 0)) with freeze_time('Monday 10th April 2017 12:00:00'): res = dao_get_paginated_most_recent_inbound_sms_by_user_number_for_service(sample_service.id, limit_days=7, page=1) # noqa diff --git a/tests/app/dao/test_jobs_dao.py b/tests/app/dao/test_jobs_dao.py index 335dece62..72e692945 100644 --- a/tests/app/dao/test_jobs_dao.py +++ b/tests/app/dao/test_jobs_dao.py @@ -136,8 +136,8 @@ def test_get_jobs_for_service_with_limit_days_param(sample_template): @freeze_time('2017-06-10') def test_get_jobs_for_service_with_limit_days_edge_case(sample_template): one_job = create_job(sample_template) - just_after_midnight_job = create_job(sample_template, created_at=datetime(2017, 6, 3, 4, 0, 1)) - just_before_midnight_job = create_job(sample_template, created_at=datetime(2017, 6, 3, 3, 59, 0)) + just_after_midnight_job = create_job(sample_template, created_at=datetime(2017, 6, 3, 0, 0, 1)) + just_before_midnight_job = create_job(sample_template, created_at=datetime(2017, 6, 2, 23, 59, 0)) jobs_limit_days = dao_get_jobs_by_service_id(one_job.service_id, limit_days=7).items assert len(jobs_limit_days) == 2 diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index d6ec13c3f..5d13bf4a4 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -777,7 +777,7 @@ def test_dao_fetch_todays_stats_for_service_only_includes_today(notify_db_sessio stats = dao_fetch_todays_stats_for_service(template.service_id) stats = {row.status: row.count for row in stats} - assert 'delivered' not in stats + assert stats['delivered'] == 1 assert stats['failed'] == 1 assert stats['created'] == 1 @@ -807,7 +807,7 @@ def test_dao_fetch_todays_stats_for_service_only_includes_today_when_clocks_spri def test_dao_fetch_todays_stats_for_service_only_includes_today_during_bst(notify_db_session): template = create_template(service=create_service()) - with freeze_time('2021-03-29T03:59:59'): + with freeze_time('2021-03-28T23:59:59'): # just before midnight BST -- not included create_notification(template=template, to_field='1', status='permanent-failure') with freeze_time('2021-03-29T04:00:01'): diff --git a/tests/app/inbound_sms/test_rest.py b/tests/app/inbound_sms/test_rest.py index 0b7cd71f2..7d6991652 100644 --- a/tests/app/inbound_sms/test_rest.py +++ b/tests/app/inbound_sms/test_rest.py @@ -97,8 +97,8 @@ def test_post_to_get_inbound_sms_allows_badly_formatted_number(admin_request, sa @freeze_time('Monday 10th April 2017 12:00') def test_post_to_get_most_recent_inbound_sms_for_service_limits_to_a_week(admin_request, sample_service): - create_inbound_sms(sample_service, created_at=datetime(2017, 4, 3, 3, 59)) - returned_inbound = create_inbound_sms(sample_service, created_at=datetime(2017, 4, 3, 4, 30)) + create_inbound_sms(sample_service, created_at=datetime(2017, 4, 2, 23, 59)) + returned_inbound = create_inbound_sms(sample_service, created_at=datetime(2017, 4, 3, 0, 30)) sms = admin_request.post('inbound_sms.post_inbound_sms_for_service', service_id=sample_service.id, _data={}) diff --git a/tests/app/job/test_rest.py b/tests/app/job/test_rest.py index c75c498e6..cca576060 100644 --- a/tests/app/job/test_rest.py +++ b/tests/app/job/test_rest.py @@ -751,10 +751,10 @@ def test_get_all_notifications_for_job_returns_csv_format(admin_request, sample_ } -@freeze_time('2017-06-10 04:00') +@freeze_time('2017-06-10 00:00') def test_get_jobs_should_retrieve_from_ft_notification_status_for_old_jobs(admin_request, sample_template): # it's the 10th today, so 3 days should include all of 7th, 8th, 9th, and some of 10th. - just_three_days_ago = datetime(2017, 6, 7, 3, 59, 59) + just_three_days_ago = datetime(2017, 6, 6, 23, 59, 59) not_quite_three_days_ago = just_three_days_ago + timedelta(seconds=1) job_1 = create_job(sample_template, created_at=just_three_days_ago, processing_started=just_three_days_ago) diff --git a/tests/app/service/test_utils.py b/tests/app/service/test_utils.py index b8f7cdd2a..1effd5308 100644 --- a/tests/app/service/test_utils.py +++ b/tests/app/service/test_utils.py @@ -4,7 +4,7 @@ # see get_financial_year for conversion of financial years. -@freeze_time("2017-04-01 03:59:59.999999") +@freeze_time("2017-03-31 23:59:59.999999") def test_get_current_financial_year_start_year_before_march(): current_fy = get_current_financial_year_start_year() assert current_fy == 2016 diff --git a/tests/app/test_model.py b/tests/app/test_model.py index e68a35c27..6ea7f772f 100644 --- a/tests/app/test_model.py +++ b/tests/app/test_model.py @@ -124,14 +124,14 @@ def test_notification_for_csv_returns_formatted_status( @freeze_time("2017-03-26 23:01:53.321312") -def test_notification_for_csv_returns_bst_correctly(sample_template): +def test_notification_for_csv_returns_utc_correctly(sample_template): notification = create_notification(sample_template) serialized = notification.serialize_for_csv() - assert serialized['created_at'] == '2017-03-26 19:01:53' + assert serialized['created_at'] == '2017-03-26 23:01:53' -def test_notification_personalisation_getter_returns_empty_dict_from_None(): +def test_notification_personalisation_getter_returns_empty_dict_from_none(): noti = Notification() noti._personalisation = None assert noti.personalisation == {} diff --git a/tests/app/test_utils.py b/tests/app/test_utils.py index 840eff607..07e36a005 100644 --- a/tests/app/test_utils.py +++ b/tests/app/test_utils.py @@ -5,28 +5,28 @@ from app.utils import ( format_sequential_number, - get_local_midnight_in_utc, get_midnight_for_day_before, + get_midnight_in_utc, midnight_n_days_ago, ) @pytest.mark.parametrize('date, expected_date', [ - (datetime(2016, 1, 15, 0, 30), datetime(2016, 1, 15, 5, 0)), - (datetime(2016, 6, 15, 0, 0), datetime(2016, 6, 15, 4, 0)), - (datetime(2016, 9, 15, 11, 59), datetime(2016, 9, 15, 4, 0)), + (datetime(2016, 1, 15, 0, 30), datetime(2016, 1, 15, 0, 0)), + (datetime(2016, 6, 15, 0, 0), datetime(2016, 6, 15, 0, 0)), + (datetime(2016, 9, 15, 11, 59), datetime(2016, 9, 15, 0, 0)), # works for both dates and datetimes - (date(2016, 1, 15), datetime(2016, 1, 15, 5, 0)), - (date(2016, 6, 15), datetime(2016, 6, 15, 4, 0)), + (date(2016, 1, 15), datetime(2016, 1, 15, 0, 0)), + (date(2016, 6, 15), datetime(2016, 6, 15, 0, 0)), ]) -def test_get_local_midnight_in_utc_returns_expected_date(date, expected_date): - assert get_local_midnight_in_utc(date) == expected_date +def test_get_midnight_in_utc_returns_expected_date(date, expected_date): + assert get_midnight_in_utc(date) == expected_date @pytest.mark.parametrize('date, expected_date', [ - (datetime(2016, 1, 15, 0, 30), datetime(2016, 1, 14, 5, 0)), - (datetime(2016, 7, 15, 0, 0), datetime(2016, 7, 14, 4, 0)), - (datetime(2016, 8, 23, 11, 59), datetime(2016, 8, 22, 4, 0)), + (datetime(2016, 1, 15, 0, 30), datetime(2016, 1, 14, 0, 0)), + (datetime(2016, 7, 15, 0, 0), datetime(2016, 7, 14, 0, 0)), + (datetime(2016, 8, 23, 11, 59), datetime(2016, 8, 22, 0, 0)), ]) def test_get_midnight_for_day_before_returns_expected_date(date, expected_date): assert get_midnight_for_day_before(date) == expected_date @@ -34,20 +34,20 @@ def test_get_midnight_for_day_before_returns_expected_date(date, expected_date): @pytest.mark.parametrize('current_time, arg, expected_datetime', [ # winter - ('2018-01-10 23:59', 1, datetime(2018, 1, 9, 5, 0)), - ('2018-01-11 00:00', 1, datetime(2018, 1, 10, 5, 0)), + ('2018-01-10 23:59', 1, datetime(2018, 1, 9, 0, 0)), + ('2018-01-11 00:00', 1, datetime(2018, 1, 10, 0, 0)), # bst switchover at 1am 25th - ('2018-03-25 10:00', 1, datetime(2018, 3, 24, 4, 0)), - ('2018-03-26 10:00', 1, datetime(2018, 3, 25, 4, 0)), - ('2018-03-27 10:00', 1, datetime(2018, 3, 26, 4, 0)), + ('2018-03-25 10:00', 1, datetime(2018, 3, 24, 0, 0)), + ('2018-03-26 10:00', 1, datetime(2018, 3, 25, 0, 0)), + ('2018-03-27 10:00', 1, datetime(2018, 3, 26, 0, 0)), # summer - ('2018-06-05 10:00', 1, datetime(2018, 6, 4, 4, 0)), + ('2018-06-05 10:00', 1, datetime(2018, 6, 4, 0, 0)), # zero days ago - ('2018-01-11 00:00', 0, datetime(2018, 1, 11, 5, 0)), - ('2018-06-05 10:00', 0, datetime(2018, 6, 5, 4, 0)), + ('2018-01-11 00:00', 0, datetime(2018, 1, 11, 0, 0)), + ('2018-06-05 10:00', 0, datetime(2018, 6, 5, 0, 0)), ]) def test_midnight_n_days_ago(current_time, arg, expected_datetime): with freeze_time(current_time): From 3b0d38ea39823f067cd0149b4ba7df7ab9aa43c5 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Tue, 30 May 2023 12:16:49 -0700 Subject: [PATCH 2/6] more fix --- app/dao/fact_notification_status_dao.py | 4 ++-- app/utils.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/dao/fact_notification_status_dao.py b/app/dao/fact_notification_status_dao.py index aa884cc2d..923ad9ad0 100644 --- a/app/dao/fact_notification_status_dao.py +++ b/app/dao/fact_notification_status_dao.py @@ -28,8 +28,8 @@ Template, ) from app.utils import ( - get_local_month_from_utc_column, get_midnight_in_utc, + get_month_from_utc_column, midnight_n_days_ago, ) @@ -358,7 +358,7 @@ def fetch_monthly_template_usage_for_service(start_date, end_date, service_id): if start_date <= datetime.utcnow() <= end_date: today = get_midnight_in_utc(datetime.utcnow()) - month = get_local_month_from_utc_column(Notification.created_at) + month = get_month_from_utc_column(Notification.created_at) stats_for_today = db.session.query( Notification.template_id.label('template_id'), diff --git a/app/utils.py b/app/utils.py index 622582997..97cc01673 100644 --- a/app/utils.py +++ b/app/utils.py @@ -63,7 +63,7 @@ def get_midnight_for_day_before(date): return get_midnight_in_utc(day_before) -def get_local_month_from_utc_column(column): +def get_month_from_utc_column(column): """ Where queries need to count notifications by month it needs to be the month in local time. @@ -75,7 +75,7 @@ def get_local_month_from_utc_column(column): """ return func.date_trunc( "month", - func.timezone(getenv("TIMEZONE", "America/New_York"), func.timezone("UTC", column)) + func.timezone(getenv("TIMEZONE", "UTC"), func.timezone("UTC", column)) ) From db0d82923890dd4f96f5146cac5456ee69a07cf5 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 31 May 2023 17:52:16 -0700 Subject: [PATCH 3/6] notify-281 fix time-sensitive tests --- app/dao/services_dao.py | 3 ++- app/service/rest.py | 1 - tests/app/celery/test_reporting_tasks.py | 2 +- tests/app/db.py | 1 - tests/app/template/test_rest_history.py | 5 +++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 4456d32b2..ea5035a8a 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -404,10 +404,11 @@ def dao_fetch_todays_stats_for_service(service_id): def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_active=True): - today = date.today() + today = datetime.utcnow().date() start_date = get_midnight_in_utc(today) end_date = get_midnight_in_utc(today + timedelta(days=1)) + subquery = db.session.query( Notification.notification_type, Notification.status, diff --git a/app/service/rest.py b/app/service/rest.py index abeb41e48..43d3ec63a 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -527,7 +527,6 @@ def get_detailed_services(start_date, end_date, only_active=False, include_from_ stats = dao_fetch_todays_stats_for_all_services(include_from_test_key=include_from_test_key, only_active=only_active) else: - stats = fetch_stats_for_all_services_by_date_range(start_date=start_date, end_date=end_date, include_from_test_key=include_from_test_key, diff --git a/tests/app/celery/test_reporting_tasks.py b/tests/app/celery/test_reporting_tasks.py index d0d6c4e3e..cbbbdefbf 100644 --- a/tests/app/celery/test_reporting_tasks.py +++ b/tests/app/celery/test_reporting_tasks.py @@ -495,7 +495,7 @@ def test_create_nightly_notification_status_for_service_and_day(notify_db_sessio def test_create_nightly_notification_status_for_service_and_day_overwrites_old_data(notify_db_session): first_service = create_service(service_name='First Service') first_template = create_template(service=first_service) - process_day = date.today() + process_day = datetime.utcnow().date() # first run: one notification, expect one row (just one status) notification = create_notification(template=first_template, status='sending') diff --git a/tests/app/db.py b/tests/app/db.py index 92320a862..30023f437 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -297,7 +297,6 @@ def create_notification( } notification = Notification(**data) dao_create_notification(notification) - return notification diff --git a/tests/app/template/test_rest_history.py b/tests/app/template/test_rest_history.py index 3e6f400d2..8f8b82ef2 100644 --- a/tests/app/template/test_rest_history.py +++ b/tests/app/template/test_rest_history.py @@ -1,5 +1,5 @@ import json -from datetime import date, datetime +from datetime import datetime from flask import url_for @@ -27,7 +27,8 @@ def test_template_history_version(notify_api, sample_user, sample_template): assert json_resp['data']['version'] == 1 assert json_resp['data']['process_type'] == 'normal' assert json_resp['data']['created_by']['name'] == sample_user.name - assert datetime.strptime(json_resp['data']['created_at'], '%Y-%m-%d %H:%M:%S.%f').date() == date.today() + assert datetime.strptime(json_resp['data']['created_at'], '%Y-%m-%d %H:%M:%S.%f').date() == \ + datetime.utcnow().date() def test_previous_template_history_version(notify_api, sample_template): From 38461af5ae8c94bad6460575c17f1efff196a04d Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 31 May 2023 17:57:24 -0700 Subject: [PATCH 4/6] fix format --- app/dao/services_dao.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index ea5035a8a..793211d80 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -408,7 +408,6 @@ def dao_fetch_todays_stats_for_all_services(include_from_test_key=True, only_act start_date = get_midnight_in_utc(today) end_date = get_midnight_in_utc(today + timedelta(days=1)) - subquery = db.session.query( Notification.notification_type, Notification.status, From 80c6c677db95c7f775407253616d42d152ab1f12 Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Wed, 31 May 2023 10:07:27 -0700 Subject: [PATCH 5/6] clean up remaining todays() --- app/dao/services_dao.py | 4 ++-- tests/app/celery/test_reporting_tasks.py | 4 ++-- tests/app/complaint/test_complaint_rest.py | 4 ++-- tests/app/dao/test_fact_billing_dao.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 793211d80..2c1d9a381 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -1,5 +1,5 @@ import uuid -from datetime import date, datetime, timedelta +from datetime import datetime, timedelta from flask import current_app from sqlalchemy import Float, cast @@ -387,7 +387,7 @@ def _delete_commit(query): def dao_fetch_todays_stats_for_service(service_id): - today = date.today() + today = datetime.utcnow().date() start_date = get_midnight_in_utc(today) return db.session.query( Notification.notification_type, diff --git a/tests/app/celery/test_reporting_tasks.py b/tests/app/celery/test_reporting_tasks.py index cbbbdefbf..f2dc6b549 100644 --- a/tests/app/celery/test_reporting_tasks.py +++ b/tests/app/celery/test_reporting_tasks.py @@ -426,7 +426,7 @@ def test_create_nightly_notification_status_for_service_and_day(notify_db_sessio second_service = create_service(service_name='second Service') second_template = create_template(service=second_service, template_type='email') - process_day = date.today() - timedelta(days=5) + process_day = datetime.utcnow().date() - timedelta(days=5) with freeze_time(datetime.combine(process_day, time.max)): create_notification(template=first_template, status='delivered') create_notification(template=second_template, status='temporary-failure') @@ -441,7 +441,7 @@ def test_create_nightly_notification_status_for_service_and_day(notify_db_sessio create_notification_history(template=second_template, status='delivered') # these created notifications from a different day get ignored - with freeze_time(datetime.combine(date.today() - timedelta(days=4), time.max)): + with freeze_time(datetime.combine(datetime.utcnow().date() - timedelta(days=4), time.max)): create_notification(template=first_template) create_notification_history(template=second_template) diff --git a/tests/app/complaint/test_complaint_rest.py b/tests/app/complaint/test_complaint_rest.py index 5f72e4ee2..9dd5aca06 100644 --- a/tests/app/complaint/test_complaint_rest.py +++ b/tests/app/complaint/test_complaint_rest.py @@ -1,5 +1,5 @@ import json -from datetime import date +from datetime import date, datetime from flask import url_for from freezegun import freeze_time @@ -74,7 +74,7 @@ def test_get_complaint_sets_start_and_end_date_to_today_if_not_specified(mocker, dao_mock = mocker.patch('app.complaint.complaint_rest.fetch_count_of_complaints', return_value=5) response = client.get(url_for('complaint.get_complaint_count'), headers=[create_admin_authorization_header()]) - dao_mock.assert_called_once_with(start_date=date.today(), end_date=date.today()) + dao_mock.assert_called_once_with(start_date=datetime.utcnow().date(), end_date=datetime.utcnow().date()) assert response.status_code == 200 assert json.loads(response.get_data(as_text=True)) == 5 diff --git a/tests/app/dao/test_fact_billing_dao.py b/tests/app/dao/test_fact_billing_dao.py index e900c0cbe..4edd7a533 100644 --- a/tests/app/dao/test_fact_billing_dao.py +++ b/tests/app/dao/test_fact_billing_dao.py @@ -714,7 +714,7 @@ def test_fetch_usage_year_for_organisation_only_queries_present_year(notify_db_s last_year = current_year - 1 date_two_years_ago = date(2021, 3, 31) date_in_last_financial_year = date(2022, 3, 31) - date_in_this_year = date.today() + date_in_this_year = datetime.utcnow().date() org = create_organisation(name='Organisation 1') From 15a70460bc38d057555675c5e7f84fa3c2a7970a Mon Sep 17 00:00:00 2001 From: Kenneth Kehl <@kkehl@flexion.us> Date: Thu, 1 Jun 2023 07:07:10 -0700 Subject: [PATCH 6/6] code review feedback --- app/dao/fact_notification_status_dao.py | 8 ++++---- app/utils.py | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/app/dao/fact_notification_status_dao.py b/app/dao/fact_notification_status_dao.py index 923ad9ad0..857256cd3 100644 --- a/app/dao/fact_notification_status_dao.py +++ b/app/dao/fact_notification_status_dao.py @@ -104,16 +104,16 @@ def fetch_notification_status_for_service_by_month(start_date, end_date, service ).all() -def fetch_notification_status_for_service_for_day(bst_day, service_id): +def fetch_notification_status_for_service_for_day(fetch_day, service_id): return db.session.query( # return current month as a datetime so the data has the same shape as the ft_notification_status query - literal(bst_day.replace(day=1), type_=DateTime).label('month'), + literal(fetch_day.replace(day=1), type_=DateTime).label('month'), Notification.notification_type, Notification.status.label('notification_status'), func.count().label('count') ).filter( - Notification.created_at >= get_midnight_in_utc(bst_day), - Notification.created_at < get_midnight_in_utc(bst_day + timedelta(days=1)), + Notification.created_at >= get_midnight_in_utc(fetch_day), + Notification.created_at < get_midnight_in_utc(fetch_day + timedelta(days=1)), Notification.service_id == service_id, Notification.key_type != KEY_TYPE_TEST ).group_by( diff --git a/app/utils.py b/app/utils.py index 97cc01673..94d4ac576 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,5 +1,4 @@ from datetime import datetime, timedelta -from os import getenv from flask import url_for from notifications_utils.template import HTMLEmailTemplate, SMSMessageTemplate @@ -75,7 +74,7 @@ def get_month_from_utc_column(column): """ return func.date_trunc( "month", - func.timezone(getenv("TIMEZONE", "UTC"), func.timezone("UTC", column)) + func.timezone("UTC", func.timezone("UTC", column)) )