Generate report from invoice data

Why these changes are being introduced:
This commit completes the refactor of
process-invoices/print_reports_by_status.py by adding functionality
to generate reports from the Alma invoice data that can be sent to
Acquisitions staff for review or as final cover sheets.

How this addresses that need:
* Adds a function in sap.py, populate_fund_data(), that completes the
  invoice record data extraction process by extracting the necessary
  invoice line fund data, combining fund amounts with identical MIT
  account numbers, and returning the extracted fund data as a dict.
* Adds a function in sap.py, generate_report(), that generates a human-
  readable report of data from an invoice data dict, structured
  according to requirements provided by stakeholders.
* Adds unit tests and updated fixtures for new functionality.
* Updates existing tests to match updated fixtures.
* Updates the sap-invoices CLI command to generate reports and log them
  if the --dry-run flag is passed. Emailing reports will be handled in
  a future commit.

Side effects of this change:
The reports generated by this process are no longer written to output
files, even during a dry run. It's possible we will want to change that
back at some point if it turns out having them as files is useful
during development, however it should not be needed in staging or
production. In general, we would prefer not to write this information
to files unless they are in a location that is excluded from version
and control AND that gets automatically purged frequently, as these
reports do contain sensitive data.

Relevant ticket(s):
* https://mitlibraries.atlassian.net/browse/IMP-2339

Author: hakbailey

llama/cli.py

@@ -150,7 +150,8 @@ def sap_invoices(ctx, dry_run, final_run):
         f"    Final run: {final_run}\n"
     )
 
-    # Retrieve and sort invoices from Alma. Log result.
+    # Retrieve and sort invoices from Alma. Log result or abort process if no invoices
+    # retrieved.
     alma_client = Alma_API_Client(config.get_alma_api_key("ALMA_API_ACQ_READ_KEY"))
     alma_client.set_content_headers("application/json", "application/json")
     invoice_records = sap.retrieve_sorted_invoices(alma_client)
@@ -163,21 +164,49 @@ def sap_invoices(ctx, dry_run, final_run):
         raise click.Abort()
 
     # For each invoice retrieved, parse and extract invoice data and save to either
-    # monograph or serials invoice list depending on invoice type. Log result.
+    # monograph or serial invoice list depending on purchase type. Log result.
     monograph_invoices = []
     serial_invoices = []
-    for invoice_record in invoice_records:
+    retrieved_vendors = {}
+    retrieved_funds = {}
+    for count, invoice_record in enumerate(invoice_records):
+        logger.info(
+            f"Extracting data for invoice #{invoice_record['id']}, "
+            f"record {count} of {len(invoice_records)}"
+        )
         invoice_data = sap.extract_invoice_data(alma_client, invoice_record)
+        vendor_code = invoice_record["vendor"]["value"]
+        try:
+            invoice_data["vendor"] = retrieved_vendors[vendor_code]
+        except KeyError:
+            logger.info(f"Retrieving data for vendor {vendor_code}")
+            retrieved_vendors[vendor_code] = sap.populate_vendor_data(
+                alma_client, vendor_code
+            )
+            invoice_data["vendor"] = retrieved_vendors[vendor_code]
+        invoice_data["funds"], retrieved_funds = sap.populate_fund_data(
+            alma_client, invoice_record, retrieved_funds
+        )
         if invoice_data["type"] == "monograph":
             monograph_invoices.append(invoice_data)
         else:
             serial_invoices.append(invoice_data)
-    logger.info(f"{len(monograph_invoices)} monograph invoices processed.")
-    logger.info(f"{len(serial_invoices)} serial invoices processed.")
+    logger.info(
+        f"{len(monograph_invoices)} monograph invoices retrieved and extracted."
+    )
+    logger.info(f"{len(serial_invoices)} serial invoices retrieved and extracted.")
 
     # Generate formatted reports for review
-
-    # If not dry run:
+    logger.info("Generating monographs report")
+    monograph_report = sap.generate_report(ctx.obj["today"], monograph_invoices)
+    logger.info("Generating serials report")
+    serial_report = sap.generate_report(ctx.obj["today"], serial_invoices)
+
+    if dry_run:
+        logger.info(f"Monograph report:\n{monograph_report}")
+        logger.info(f"Serials report:\n{serial_report}")
+    else:
+        pass
     # Send email with reports as attachments (note that email subject and attachment
     # file names will differ for review vs. final run)
 
@@ -191,3 +220,10 @@ def sap_invoices(ctx, dry_run, final_run):
     # Send data and control files to SAP dropbox via SFTP
     # Email summary files
     # Update invoice statuses in Alma
+
+    logger.info(
+        "SAP invoice process completed for a review run:\n"
+        f"    {len(monograph_invoices)} monograph invoices retrieved and processed\n"
+        f"    {len(serial_invoices)} serial invoices retrieved and processed\n"
+        f"    {len(invoice_records)} total invoices retrieved and processed\n"
+    )

llama/sap.py

@@ -3,6 +3,7 @@
 import json
 import logging
 from datetime import datetime
+from typing import List, Tuple
 
 from llama.alma import Alma_API_Client
 
@@ -32,11 +33,11 @@ def extract_invoice_data(alma_client: Alma_API_Client, invoice_record: dict) ->
     invoice_data = {
         "date": datetime.strptime(invoice_record["invoice_date"], "%Y-%m-%dZ"),
         "id": invoice_record["id"],
+        "number": invoice_record["number"],
         "type": purchase_type(vendor_code),
         "payment method": invoice_record["payment_method"]["value"],
         "total amount": invoice_record["total_amount"],
         "currency": invoice_record["currency"]["value"],
-        "vendor": populate_vendor_data(alma_client, vendor_code),
     }
     return invoice_data
 
@@ -115,3 +116,89 @@ def country_code_from_address(address: dict) -> str:
         return COUNTRIES[country]
     except KeyError:
         return "US"
+
+
+def populate_fund_data(
+    alma_client: Alma_API_Client, invoice_record: dict, retrieved_funds: dict
+) -> Tuple[dict, dict]:
+    """Populate a dict with fund data needed for SAP.
+
+    Given an invoice record, a dict of already retrieved funds, and an authenticated
+    Alma client, return a dict populated with the fund data needed for SAP.
+
+    Note: Also returns a dict of all fund records retrieved from Alma so we can pass
+    that to subsequent calls to this function. That way we only call the Alma API once
+    throughout the entire process for each fund we need, rather than retrieving the
+    same fund record every time the fund appears in an invoice.
+    """
+    fund_data = {}
+    invoice_lines_total = 0
+    for invoice_line in invoice_record["invoice_lines"]["invoice_line"]:
+        for fund_distribution in invoice_line["fund_distribution"]:
+            fund_code = fund_distribution["fund_code"]["value"]
+            amount = fund_distribution["amount"]
+            try:
+                fund_record = retrieved_funds[fund_code]
+            except KeyError:
+                logger.info(f"Retrieving data for fund {fund_code}")
+                retrieved_funds[fund_code] = alma_client.get_fund_by_code(fund_code)
+                fund_record = retrieved_funds[fund_code]
+            external_id = fund_record["fund"][0]["external_id"].strip()
+            try:
+                # Combine amounts for funds that have the same external ID (AKA the
+                # same MIT G/L account and cost object)
+                fund_data[external_id]["amount"] += amount
+            except KeyError:
+                fund_data[external_id] = {
+                    "amount": amount,
+                    "G/L account": external_id.split("-")[0],
+                    "cost object": external_id.split("-")[1],
+                }
+            invoice_lines_total += amount
+    return fund_data, retrieved_funds
+
+
+def generate_report(today: datetime, invoices: List[dict]) -> str:
+    today_string = today.strftime("%m/%d/%Y")
+    report = ""
+    for invoice in invoices:
+        report += f"\n\n{'':33}MIT LIBRARIES\n\n\n"
+        report += (
+            f"Date: {today_string:<36}Vendor code   : {invoice['vendor']['code']}\n"
+        )
+        report += f"{'Accounting ID :':>57}\n\n"
+        report += f"Vendor:  {invoice['vendor']['name']}\n"
+        for line in invoice["vendor"]["address"]["lines"]:
+            report += f"         {line}\n"
+        report += "         "
+        if invoice["vendor"]["address"]["city"]:
+            report += f"{invoice['vendor']['address']['city']}, "
+        if invoice["vendor"]["address"]["state or province"]:
+            report += f"{invoice['vendor']['address']['state or province']} "
+        if invoice["vendor"]["address"]["postal code"]:
+            report += f"{invoice['vendor']['address']['postal code']}"
+        report += f"\n         {invoice['vendor']['address']['country']}\n\n"
+        report += (
+            "Invoice no.            Fiscal Account     Amount            Inv. Date\n"
+        )
+        report += (
+            "------------------     -----------------  -------------     ----------\n"
+        )
+        for fund in invoice["funds"]:
+            report += f"{invoice['number'] + invoice['date'].strftime('%y%m%d'):<23}"
+            report += (
+                f"{invoice['funds'][fund]['G/L account']} "
+                f"{invoice['funds'][fund]['cost object']}     "
+            )
+            report += f"{invoice['funds'][fund]['amount']:<18,.2f}"
+            report += f"{invoice['date'].strftime('%m/%d/%Y')}\n"
+        report += "\n\n"
+        report += (
+            f"Total/Currency:             {invoice['total amount']:,.2f}      "
+            f"{invoice['currency']}\n\n"
+        )
+        report += f"Payment Method:  {invoice['payment method']}\n\n\n"
+        report += f"{'Departmental Approval':>44} {'':_<34}\n\n"
+        report += f"{'Financial Services Approval':>50} {'':_<28}\n\n\n"
+        report += "\f"
+    return report

tests/conftest.py

@@ -35,7 +35,23 @@ def mocked_alma(po_line_record_all_fields):
             ]
         }
         with open("tests/fixtures/funds.json") as f:
-            m.get("http://example.com/acq/funds", json=json.load(f))
+            funds = json.load(f)
+            m.get(
+                "http://example.com/acq/funds?q=fund_code~ABC",
+                json={"fund": [funds["fund"][0]]},
+            )
+            m.get(
+                "http://example.com/acq/funds?q=fund_code~DEF",
+                json={"fund": [funds["fund"][1]]},
+            )
+            m.get(
+                "http://example.com/acq/funds?q=fund_code~GHI",
+                json={"fund": [funds["fund"][2]]},
+            )
+            m.get(
+                "http://example.com/acq/funds?q=fund_code~JKL",
+                json={"fund": [funds["fund"][3]]},
+            )
         with open("tests/fixtures/invoices.json") as f:
             m.get(
                 "http://example.com/acq/invoices/0501130657",
@@ -154,7 +170,7 @@ def po_line_record_multiple_funds():
         "created_date": "2021-05-13Z",
         "fund_distribution": [
             {"fund_code": {"value": "ABC"}, "amount": {"sum": "6.0"}},
-            {"fund_code": {"value": "DEF"}, "amount": {"sum": "6.0"}},
+            {"fund_code": {"value": "GHI"}, "amount": {"sum": "6.0"}},
         ],
         "note": [{"note_text": ""}],
     }

tests/fixtures/funds.json

@@ -1,11 +1,146 @@
 {
-  "total_record_count": 999,
+  "total_record_count": 4,
   "fund": [
     {
       "link": "string",
       "code": "ABC",
       "name": "Biology",
-      "external_id": "1239410001021",
+      "external_id": "1234567-000001",
+      "type": {
+        "value": "General"
+      },
+      "entity_type": {
+        "value": "SUMMARY"
+      },
+      "owner": {
+        "value": "LAW"
+      },
+      "description": "desc",
+      "fiscal_period": {
+        "value": "29"
+      },
+      "currency": {
+        "value": "USD"
+      },
+      "available_for_library": [
+        {
+          "value": "string"
+        }
+      ],
+      "parent": {
+        "value": "43417015650001021"
+      },
+      "overencumbrance_allowed": {
+        "value": "yes"
+      },
+      "overexpenditure_allowed": {
+        "value": "no"
+      },
+      "overencumbrance_warning_percent": 15,
+      "overexpenditure_warning_sum": 17.44,
+      "overencumbrance_limit_percent": 8,
+      "overexpenditure_limit_sum": 10.44,
+      "encumbrances_prior_to_fiscal_period": 11,
+      "expenditures_prior_to_fiscal_period": 16,
+      "transfers_prior_to_fiscal_period": 12,
+      "fiscal_period_end_encumbrance_grace_period": 5,
+      "fiscal_period_end_expenditure_grace_period": 8
+    },
+    {
+      "link": "string",
+      "code": "DEF",
+      "name": "Fund DEF",
+      "external_id": "1234567-000001",
+      "type": {
+        "value": "General"
+      },
+      "entity_type": {
+        "value": "SUMMARY"
+      },
+      "owner": {
+        "value": "LAW"
+      },
+      "description": "desc",
+      "fiscal_period": {
+        "value": "29"
+      },
+      "currency": {
+        "value": "USD"
+      },
+      "available_for_library": [
+        {
+          "value": "string"
+        }
+      ],
+      "parent": {
+        "value": "43417015650001021"
+      },
+      "overencumbrance_allowed": {
+        "value": "yes"
+      },
+      "overexpenditure_allowed": {
+        "value": "no"
+      },
+      "overencumbrance_warning_percent": 15,
+      "overexpenditure_warning_sum": 17.44,
+      "overencumbrance_limit_percent": 8,
+      "overexpenditure_limit_sum": 10.44,
+      "encumbrances_prior_to_fiscal_period": 11,
+      "expenditures_prior_to_fiscal_period": 16,
+      "transfers_prior_to_fiscal_period": 12,
+      "fiscal_period_end_encumbrance_grace_period": 5,
+      "fiscal_period_end_expenditure_grace_period": 8
+    },
+    {
+      "link": "string",
+      "code": "GHI",
+      "name": "Fund GHI",
+      "external_id": "1234567-000002",
+      "type": {
+        "value": "General"
+      },
+      "entity_type": {
+        "value": "SUMMARY"
+      },
+      "owner": {
+        "value": "LAW"
+      },
+      "description": "desc",
+      "fiscal_period": {
+        "value": "29"
+      },
+      "currency": {
+        "value": "USD"
+      },
+      "available_for_library": [
+        {
+          "value": "string"
+        }
+      ],
+      "parent": {
+        "value": "43417015650001021"
+      },
+      "overencumbrance_allowed": {
+        "value": "yes"
+      },
+      "overexpenditure_allowed": {
+        "value": "no"
+      },
+      "overencumbrance_warning_percent": 15,
+      "overexpenditure_warning_sum": 17.44,
+      "overencumbrance_limit_percent": 8,
+      "overexpenditure_limit_sum": 10.44,
+      "encumbrances_prior_to_fiscal_period": 11,
+      "expenditures_prior_to_fiscal_period": 16,
+      "transfers_prior_to_fiscal_period": 12,
+      "fiscal_period_end_encumbrance_grace_period": 5,
+      "fiscal_period_end_expenditure_grace_period": 8
+    },
+    {
+      "link": "string",
+      "code": "JKL",
+      "name": "Fund JKL",
+      "external_id": "1234567-000003",
       "type": {
         "value": "General"
       },

tests/test_alma.py

@@ -33,7 +33,7 @@ def test_alma_get_brief_po_lines(mocked_alma, mocked_alma_api_client):
 def test_alma_get_fund_by_code(mocked_alma, mocked_alma_api_client):
     fund = mocked_alma_api_client.get_fund_by_code("ABC")
     assert fund["fund"][0]["code"] == "ABC"
-    assert fund["fund"][0]["external_id"] == "1239410001021"
+    assert fund["fund"][0]["external_id"] == "1234567-000001"
 
 
 def test_alma_get_invoice(mocked_alma, mocked_alma_api_client):