(PC-9939) payments: Always store payment files on disk

That way, it's easier to manually put them all on Google Drive. (And,
when files are automatically sent there, we will do the same and send
all files, whether e-mails were successfully sent or not.)

src/pcapi/scripts/payment/batch_steps.py

@@ -179,7 +179,14 @@ def send_transactions(
     )
     logger.info("[BATCH][PAYMENTS] Recipients of email : %s", recipients)
 
-    send_payment_message_email(xml_file, venues_csv, checksum, recipients)
+    venues_csv_path = _save_file_on_disk("venues", venues_csv, "csv")
+    xml_path = _save_file_on_disk("banque_de_france", xml_file, "csv")
+    if not send_payment_message_email(xml_file, venues_csv, checksum, recipients):
+        logger.info(
+            "[BATCH][PAYMENTS] Could not send payment message email. Files have been stored at %s and %s",
+            venues_csv_path,
+            xml_path,
+        )
     logger.info("[BATCH][PAYMENTS] Updating status of payments to UNDER_REVIEW")
     payments_api.bulk_create_payment_statuses(payment_query, TransactionStatus.UNDER_REVIEW, detail=None)
 
@@ -196,11 +203,11 @@ def send_payments_details(payment_query, recipients: list[str]) -> None:
     csv = generate_payment_details_csv(payment_query)
     logger.info("[BATCH][PAYMENTS] Sending CSV details of %s payments", count)
     logger.info("[BATCH][PAYMENTS] Recipients of email : %s", recipients)
+    path = _save_file_on_disk("payments_details", csv, "csv")
     if not send_payment_details_email(csv, recipients):
         # FIXME (dbaty, 2021-06-16): we are likely to end up here
         # because the attachment is now over Mailjet's 15Mb limit.
         # This is an ugly quick fix.
-        path = _save_file_on_disk("payments_details", csv)
         logger.info("[BATCH][PAYMENTS] Could not send payment details email. CSV file has been stored at %s", path)
 
 
@@ -218,9 +225,9 @@ def send_wallet_balances(recipients: list[str]) -> None:
         logger.exception("[BATCH][PAYMENTS] Error while sending users wallet balances email to MailJet: %s", exception)
 
 
-def _save_file_on_disk(filename_prefix: str, content: str) -> pathlib.Path:
+def _save_file_on_disk(filename_prefix: str, content: str, extension: str) -> pathlib.Path:
     dt = datetime.now().strftime("%Y%m%d_%H%M%S")
-    path = pathlib.Path(tempfile.gettempdir()) / f"{filename_prefix}_{dt}.csv"
+    path = pathlib.Path(tempfile.gettempdir()) / f"{filename_prefix}_{dt}.{extension}"
     path.write_text(content, encoding="utf-8")
     return path
 
@@ -240,11 +247,11 @@ def send_payments_report(batch_date: datetime, recipients: list[str]) -> None:
 
     n_payments_by_status = payment_queries.get_payment_count_by_status(batch_date)
 
+    path = _save_file_on_disk("payments_not_processable", not_processable_csv, "csv")
     if not send_payments_report_emails(not_processable_csv, n_payments_by_status, recipients):
         # FIXME (dbaty, 2021-06-16): we are likely to end up here
         # because the attachment is now over Mailjet's 15Mb limit.
         # This is an ugly quick fix.
-        path = _save_file_on_disk("payments_not_processable", not_processable_csv)
         logger.info("[BATCH][PAYMENTS] Could not send payment reports email. CSV file has been stored at %s", path)
 
 

tests/scripts/payment/batch_steps_test.py

@@ -1,6 +1,4 @@
 import datetime
-import pathlib
-import tempfile
 
 from lxml.etree import DocumentInvalid
 import pytest
@@ -185,27 +183,6 @@ def test_send_payments_details_sends_a_csv_attachment():
     assert mails_testing.outbox[0].sent_data["Attachments"][0]["ContentType"] == "application/zip"
 
 
-@pytest.mark.usefixtures("db_session")
-@override_settings(EMAIL_BACKEND="pcapi.core.mails.backends.testing.FailingBackend")
-def test_send_payments_details_fallbacks_to_stored_csv_on_mail_error():
-    iban = "CF13QSDFGH456789"
-    bic = "AZERTY9Q666"
-    payments_factories.PaymentFactory(iban=iban, bic=bic, recipientName="Testé")
-
-    # FIXME (dbaty, 2021-06-16): this is ugly and I know it. (Quick
-    # naive idea: write a context manager with a try/finally that
-    # mocks `tempfile.getttempdir()` with a custom, new directory, and
-    # clean it up on exit).
-    tmp_dir = pathlib.Path(tempfile.gettempdir())
-    files = set(tmp_dir.glob("payments_details_*.csv"))
-    send_payments_details(Payment.query, ["test@example.com"])
-    new_files = set(tmp_dir.glob("payments_details_*.csv")) - files
-    assert len(new_files) == 1
-    new_file = new_files.pop()
-    header = new_file.read_text().splitlines()[0]
-    assert header.startswith('"Libellé fournisseur","Raison sociale de la structure","SIREN"')
-
-
 @pytest.mark.usefixtures("db_session")
 def test_send_payment_details_does_not_send_anything_if_all_payment_have_error_status():
     send_payments_details(Payment.query, ["comptable@test.com"])
@@ -257,26 +234,6 @@ def test_send_payments_report_sends_one_csv_attachment_if_some_payments_are_not_
     assert mails_testing.outbox[0].sent_data["Attachments"][0]["ContentType"] == "text/csv"
 
 
-@pytest.mark.usefixtures("db_session")
-@override_settings(EMAIL_BACKEND="pcapi.core.mails.backends.testing.FailingBackend")
-def test_send_payments_report_fallbacks_to_stored_csv_on_mail_error():
-    batch_date = datetime.datetime.now()
-    payments_factories.PaymentStatusFactory(status=TransactionStatus.NOT_PROCESSABLE, payment__batchDate=batch_date)
-
-    # FIXME (dbaty, 2021-06-16): this is ugly and I know it. (Quick
-    # naive idea: write a context manager with a try/finally that
-    # mocks `tempfile.getttempdir()` with a custom, new directory, and
-    # clean it up on exit).
-    tmp_dir = pathlib.Path(tempfile.gettempdir())
-    files = set(tmp_dir.glob("payments_not_processable_*.csv"))
-    send_payments_report(batch_date, ["recipient@example.com"])
-    new_files = set(tmp_dir.glob("payments_not_processable_*.csv")) - files
-    assert len(new_files) == 1
-    new_file = new_files.pop()
-    header = new_file.read_text().splitlines()[0]
-    assert header.startswith('"Libellé fournisseur","Raison sociale de la structure","SIREN"')
-
-
 class SetNotProcessablePaymentsWithBankInformationToRetryTest:
     @pytest.mark.usefixtures("db_session")
     def test_should_set_not_processable_payments_to_retry_and_update_payments_bic_and_iban_using_offerer_information(