From de0ca845a5de215fe4b5c14fd07842e29866738f Mon Sep 17 00:00:00 2001 From: Damien Baty Date: Wed, 7 Jul 2021 21:10:55 +0200 Subject: [PATCH] (PC-9939) payments: Always store payment files on disk That way, it's easier to manually put them all on Google Drive. (And, when files are automatically sent there, we will do the same and send all files, whether e-mails were successfully sent or not.) --- src/pcapi/scripts/payment/batch_steps.py | 17 ++++++--- tests/scripts/payment/batch_steps_test.py | 43 ----------------------- 2 files changed, 12 insertions(+), 48 deletions(-) diff --git a/src/pcapi/scripts/payment/batch_steps.py b/src/pcapi/scripts/payment/batch_steps.py index 4badc39e77..2a12a31f2a 100644 --- a/src/pcapi/scripts/payment/batch_steps.py +++ b/src/pcapi/scripts/payment/batch_steps.py @@ -179,7 +179,14 @@ def send_transactions( ) logger.info("[BATCH][PAYMENTS] Recipients of email : %s", recipients) - send_payment_message_email(xml_file, venues_csv, checksum, recipients) + venues_csv_path = _save_file_on_disk("venues", venues_csv, "csv") + xml_path = _save_file_on_disk("banque_de_france", xml_file, "csv") + if not send_payment_message_email(xml_file, venues_csv, checksum, recipients): + logger.info( + "[BATCH][PAYMENTS] Could not send payment message email. Files have been stored at %s and %s", + venues_csv_path, + xml_path, + ) logger.info("[BATCH][PAYMENTS] Updating status of payments to UNDER_REVIEW") payments_api.bulk_create_payment_statuses(payment_query, TransactionStatus.UNDER_REVIEW, detail=None) @@ -196,11 +203,11 @@ def send_payments_details(payment_query, recipients: list[str]) -> None: csv = generate_payment_details_csv(payment_query) logger.info("[BATCH][PAYMENTS] Sending CSV details of %s payments", count) logger.info("[BATCH][PAYMENTS] Recipients of email : %s", recipients) + path = _save_file_on_disk("payments_details", csv, "csv") if not send_payment_details_email(csv, recipients): # FIXME (dbaty, 2021-06-16): we are likely to end up here # because the attachment is now over Mailjet's 15Mb limit. # This is an ugly quick fix. - path = _save_file_on_disk("payments_details", csv) logger.info("[BATCH][PAYMENTS] Could not send payment details email. CSV file has been stored at %s", path) @@ -218,9 +225,9 @@ def send_wallet_balances(recipients: list[str]) -> None: logger.exception("[BATCH][PAYMENTS] Error while sending users wallet balances email to MailJet: %s", exception) -def _save_file_on_disk(filename_prefix: str, content: str) -> pathlib.Path: +def _save_file_on_disk(filename_prefix: str, content: str, extension: str) -> pathlib.Path: dt = datetime.now().strftime("%Y%m%d_%H%M%S") - path = pathlib.Path(tempfile.gettempdir()) / f"{filename_prefix}_{dt}.csv" + path = pathlib.Path(tempfile.gettempdir()) / f"{filename_prefix}_{dt}.{extension}" path.write_text(content, encoding="utf-8") return path @@ -240,11 +247,11 @@ def send_payments_report(batch_date: datetime, recipients: list[str]) -> None: n_payments_by_status = payment_queries.get_payment_count_by_status(batch_date) + path = _save_file_on_disk("payments_not_processable", not_processable_csv, "csv") if not send_payments_report_emails(not_processable_csv, n_payments_by_status, recipients): # FIXME (dbaty, 2021-06-16): we are likely to end up here # because the attachment is now over Mailjet's 15Mb limit. # This is an ugly quick fix. - path = _save_file_on_disk("payments_not_processable", not_processable_csv) logger.info("[BATCH][PAYMENTS] Could not send payment reports email. CSV file has been stored at %s", path) diff --git a/tests/scripts/payment/batch_steps_test.py b/tests/scripts/payment/batch_steps_test.py index c0b195a48f..b6777c22c6 100644 --- a/tests/scripts/payment/batch_steps_test.py +++ b/tests/scripts/payment/batch_steps_test.py @@ -1,6 +1,4 @@ import datetime -import pathlib -import tempfile from lxml.etree import DocumentInvalid import pytest @@ -185,27 +183,6 @@ def test_send_payments_details_sends_a_csv_attachment(): assert mails_testing.outbox[0].sent_data["Attachments"][0]["ContentType"] == "application/zip" -@pytest.mark.usefixtures("db_session") -@override_settings(EMAIL_BACKEND="pcapi.core.mails.backends.testing.FailingBackend") -def test_send_payments_details_fallbacks_to_stored_csv_on_mail_error(): - iban = "CF13QSDFGH456789" - bic = "AZERTY9Q666" - payments_factories.PaymentFactory(iban=iban, bic=bic, recipientName="Testé") - - # FIXME (dbaty, 2021-06-16): this is ugly and I know it. (Quick - # naive idea: write a context manager with a try/finally that - # mocks `tempfile.getttempdir()` with a custom, new directory, and - # clean it up on exit). - tmp_dir = pathlib.Path(tempfile.gettempdir()) - files = set(tmp_dir.glob("payments_details_*.csv")) - send_payments_details(Payment.query, ["test@example.com"]) - new_files = set(tmp_dir.glob("payments_details_*.csv")) - files - assert len(new_files) == 1 - new_file = new_files.pop() - header = new_file.read_text().splitlines()[0] - assert header.startswith('"Libellé fournisseur","Raison sociale de la structure","SIREN"') - - @pytest.mark.usefixtures("db_session") def test_send_payment_details_does_not_send_anything_if_all_payment_have_error_status(): send_payments_details(Payment.query, ["comptable@test.com"]) @@ -257,26 +234,6 @@ def test_send_payments_report_sends_one_csv_attachment_if_some_payments_are_not_ assert mails_testing.outbox[0].sent_data["Attachments"][0]["ContentType"] == "text/csv" -@pytest.mark.usefixtures("db_session") -@override_settings(EMAIL_BACKEND="pcapi.core.mails.backends.testing.FailingBackend") -def test_send_payments_report_fallbacks_to_stored_csv_on_mail_error(): - batch_date = datetime.datetime.now() - payments_factories.PaymentStatusFactory(status=TransactionStatus.NOT_PROCESSABLE, payment__batchDate=batch_date) - - # FIXME (dbaty, 2021-06-16): this is ugly and I know it. (Quick - # naive idea: write a context manager with a try/finally that - # mocks `tempfile.getttempdir()` with a custom, new directory, and - # clean it up on exit). - tmp_dir = pathlib.Path(tempfile.gettempdir()) - files = set(tmp_dir.glob("payments_not_processable_*.csv")) - send_payments_report(batch_date, ["recipient@example.com"]) - new_files = set(tmp_dir.glob("payments_not_processable_*.csv")) - files - assert len(new_files) == 1 - new_file = new_files.pop() - header = new_file.read_text().splitlines()[0] - assert header.startswith('"Libellé fournisseur","Raison sociale de la structure","SIREN"') - - class SetNotProcessablePaymentsWithBankInformationToRetryTest: @pytest.mark.usefixtures("db_session") def test_should_set_not_processable_payments_to_retry_and_update_payments_bic_and_iban_using_offerer_information(