Build framework for creating reports

Makefile

@@ -48,8 +48,7 @@ safety: # Check for security vulnerabilities and verify Pipfile.lock is up-to-da
 	pipenv check
 	pipenv verify
 
-lint-apply: # Apply changes with 'black' and resolve 'fixable errors' with 'ruff'
-	black-apply ruff-apply 
+lint-apply: black-apply ruff-apply # Apply changes with 'black' and resolve 'fixable errors' with 'ruff'
 
 black-apply: # Apply changes with 'black'
 	pipenv run black .

Pipfile

@@ -7,12 +7,14 @@ name = "pypi"
 boto3 = "*"
 boto3-stubs = {extras = ["essential","ses"], version = "*"}
 click = "*"
+jinja2 = "*"
 sentry-sdk = "*"
 smart_open = {extras = ["s3"], version = "*"}
 
 [dev-packages]
 black = "*"
 coveralls = "*"
+freezegun = "*"
 moto="*"
 mypy = "*"
 pre-commit = "*"

dsc/cli.py

@@ -6,6 +6,7 @@
 
 from dsc.config import Config
 from dsc.exceptions import ReconcileError
+from dsc.reports import FinalizeReport
 from dsc.workflows.base import Workflow
 
 logger = logging.getLogger(__name__)
@@ -74,8 +75,10 @@ def reconcile(ctx: click.Context) -> None:
     workflow = ctx.obj["workflow"]
     try:
         workflow.reconcile_bitstreams_and_metadata()
+        # TODO(): workflow.send_report(email_recipients.split(",")) #noqa:FIX002, TD003
     except ReconcileError:
         logger.info("Reconcile failed.")
+        # TODO(): workflow.send_report(email_recipients.split(",")) #noqa:FIX002, TD003
         ctx.exit(1)
 
 
@@ -102,7 +105,7 @@ def submit(
     workflow = ctx.obj["workflow"]
     logger.debug(f"Beginning submission of batch ID: {workflow.batch_id}")
     workflow.submit_items(collection_handle)
-    # TODO(): workflow.report_results(email_recipients.split(",")) #noqa:FIX002, TD003
+    # TODO(): workflow.send_report(email_recipients.split(",")) #noqa:FIX002, TD003
 
 
 @main.command()
@@ -117,4 +120,6 @@ def finalize(ctx: click.Context, email_recipients: str) -> None:
     """Process the result messages from the DSS output queue according the workflow."""
     workflow = ctx.obj["workflow"]
     workflow.process_results()
-    workflow.report_results(email_recipients.split(","))
+    workflow.send_report(
+        report_class=FinalizeReport, email_recipients=email_recipients.split(",")
+    )

dsc/reports/__init__.py

@@ -0,0 +1,4 @@
+from dsc.reports.base import Report
+from dsc.reports.finalize import FinalizeReport
+
+__all__ = ["FinalizeReport", "Report"]

dsc/reports/base.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import datetime
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+from jinja2 import Environment, FileSystemLoader, Template, select_autoescape
+
+if TYPE_CHECKING:
+    import dsc.workflows as workflows  # noqa: PLR0402
+
+
+class Report(ABC):
+    """A base report class from which other report classes are derived."""
+
+    def __init__(
+        self, workflow_name: str, batch_id: str, events: workflows.WorkflowEvents
+    ):
+        self.workflow_name = workflow_name
+        self.batch_id = batch_id
+        self.report_date = datetime.datetime.now(tz=datetime.UTC).strftime(
+            "%Y-%m-%d %H:%M:%S"
+        )
+        self.events = events
+
+        # configure environment for loading jinja templates
+        self.jinja_env = Environment(
+            loader=FileSystemLoader(["dsc/templates/html", "dsc/templates/plain_text"]),
+            autoescape=select_autoescape(),
+        )
+
+    @property
+    @abstractmethod
+    def jinja_template_plain_text_filename(self) -> str:
+        """Plain-text template filename."""
+
+    @property
+    @abstractmethod
+    def jinja_template_html_filename(self) -> str:
+        """HTML template filename."""
+
+    @property
+    def jinja_template_plain_text(self) -> Template:
+        return self.jinja_env.get_template(self.jinja_template_plain_text_filename)
+
+    @property
+    def jinja_template_html(self) -> Template:
+        return self.jinja_env.get_template(self.jinja_template_html_filename)
+
+    @property
+    @abstractmethod
+    def subject(self) -> str:
+        """Subject heading used in report email."""
+
+    @classmethod
+    def from_workflow(cls, workflow: workflows.Workflow) -> Report:
+        """Create instance of Report using dsc.workflows.Workflow."""
+        return cls(
+            workflow_name=workflow.workflow_name,
+            batch_id=workflow.batch_id,
+            events=workflow.workflow_events,
+        )
+
+    @abstractmethod
+    def create_attachments(self) -> list[tuple]:
+        """Create attachments to include in report email."""
+
+    @abstractmethod
+    def to_plain_text(self) -> str:
+        """Render plain-text template."""
+
+    @abstractmethod
+    def to_html(self) -> str:
+        """Render HTML template."""

dsc/reports/finalize.py

@@ -0,0 +1,123 @@
+import csv
+from io import StringIO
+
+from dsc.reports.base import Report
+
+
+class FinalizeReport(Report):
+    """Report class for 'finalize' methods.
+
+    This report is used to create an email summarizing the results
+    from running the 'finalize' methods, which processes the result
+    messages from DSpace Submission Service (DSS) sent to the output
+    queue for a given workflow.
+
+    The email created by this report is structured as follows:
+
+    1. A message summarizing the number of successfully deposited items
+       and the number of encountered errors.
+
+    2. A CSV file included as an attachment describing successfully deposited items,
+       which consists of the columns: 'item_identifier' and 'dspace_handle' (i.e.,
+       the 'ItemHandle' from the DSS result message). Created only if
+       any items in Workflow.processed_items have ingested="success".
+
+    3. A text file included as an attachment logging all errors encountered when
+       'finalize' methods were executed. Created only if any WorkflowEvents.errors exist.
+    """
+
+    @property
+    def jinja_template_plain_text_filename(self) -> str:
+        """Plain-text template filename."""
+        return "finalize.txt"
+
+    @property
+    def jinja_template_html_filename(self) -> str:
+        """HTML template filename."""
+        return "finalize.html"
+
+    @property
+    def subject(self) -> str:
+        return (
+            f"DSpace Submission Results - {self.workflow_name}, batch='{self.batch_id}'"
+        )
+
+    def create_attachments(self) -> list[tuple]:
+        """Create file attachments for 'finalize' email.
+
+        This method will create a CSV file of successfully deposited
+        items and optionally create a text file of error messages.
+        """
+        attachments = []
+
+        ingested_items = self.get_ingested_items()
+        if ingested_items:
+            attachments.append(
+                (
+                    "ingested_items.csv",
+                    self._write_ingested_items_csv(ingested_items),
+                )
+            )
+
+        if self.events.errors:
+            attachments.append(
+                (
+                    "errors.txt",
+                    self._write_errors_text_file(),
+                )
+            )
+        return attachments
+
+    def get_ingested_items(self) -> list[dict]:
+        return [
+            {
+                "item_identifier": item["item_identifier"],
+                "dspace_handle": item["result_message_body"]["ItemHandle"],
+            }
+            for item in self.events.processed_items
+            if item["ingested"] == "success"
+        ]
+
+    def _write_ingested_items_csv(self, ingested_items: list[dict]) -> StringIO:
+        """Write ingested items to string buffer.
+
+        This method creates a string buffer with the contents of a CSV
+        file describing successfully ingested items.
+        """
+        csv_buffer = StringIO()
+        fieldnames = ingested_items[0].keys()
+        writer = csv.DictWriter(csv_buffer, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(ingested_items)
+        csv_buffer.seek(0)
+        return csv_buffer
+
+    def _write_errors_text_file(self) -> StringIO:
+        """Write error messages to string buffer.
+
+        This method creates a string buffer with the error messages
+        encountered when 'finalize' methods were executed.
+        """
+        text_buffer = StringIO()
+        for error in self.events.errors:
+            text_buffer.write(error + "\n")
+        text_buffer.seek(0)
+        return text_buffer
+
+    def to_plain_text(self) -> str:
+        return self.jinja_template_plain_text.render(
+            workflow_name=self.workflow_name,
+            batch_id=self.batch_id,
+            report_date=self.report_date,
+            processed_items=self.events.processed_items,
+            errors=self.events.errors,
+        )
+
+    def to_html(self) -> str:
+        return self.jinja_template_html.render(
+            workflow_name=self.workflow_name,
+            batch_id=self.batch_id,
+            report_date=self.report_date,
+            processed_items=self.events.processed_items,
+            errors=self.events.errors,
+        )

dsc/templates/html/base.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en-US">
+    <head>
+        <title></title>
+    </head>
+    <body>
+        <p>Hello,</p>
+
+        <div id="content">{% block content %}{% endblock %}</div>
+    </body>
+
+</html>
+

dsc/templates/html/finalize.html

@@ -0,0 +1,12 @@
+{% extends "base.html" %}
+
+{% block content %}
+
+<p><b>Results summary for {{workflow_name}} deposit for batch='{{batch_id}}'</b></p>
+<p>Run date: {{report_date}}</p>
+<p>Results:</p>
+
+<p>Ingested: {{ processed_items|length }}</p>
+<p>Errors: {{ errors|length }}</p>
+
+{% endblock %}

dsc/templates/plain_text/base.txt

@@ -0,0 +1,2 @@
+Hello, 
+{% block content %}{% endblock %}

dsc/templates/plain_text/finalize.txt

@@ -0,0 +1,8 @@
+{% extends "base.txt" %}
+
+Results summary for {{workflow_name}} deposit for batch='{{batch_id}}'
+Run date: {{report_date}}
+Results:
+
+Ingested: {{ processed_items|length }}
+Errors: {{ errors|length }}

dsc/utilities/aws/ses.py

@@ -3,11 +3,14 @@
 import logging
 from email.mime.application import MIMEApplication
 from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
 from typing import TYPE_CHECKING
 
 from boto3 import client
 
 if TYPE_CHECKING:  # pragma: no cover
+    from io import StringIO
+
     from mypy_boto3_ses.type_defs import SendRawEmailResponseTypeDef
 
 logger = logging.getLogger(__name__)
@@ -22,46 +25,55 @@ def __init__(self, region: str) -> None:
     def create_and_send_email(
         self,
         subject: str,
-        attachment_content: str,
-        attachment_name: str,
         source_email_address: str,
         recipient_email_addresses: list[str],
+        message_body_plain_text: str,
+        message_body_html: str | None = None,
+        attachments: list[tuple] | None = None,
     ) -> None:
         """Create an email message and send it via SES.
 
         Args:
-           subject: The subject of the email.
-           attachment_content: The content of the email attachment.
-           attachment_name: The name of the email attachment.
-           source_email_address: The email address of the sender.
-           recipient_email_addresses: The email address of the receipient.
+            subject: The subject of the email.
+            source_email_address: The email address of the sender.
+            recipient_email_addresses: The email address of the receipient.
+            message_body_plain_text: Message body rendered in plain-text.
+            message_body_html: Message body rendered in HTML.
+            attachments: Attachments to include in an email, represented as
+                a list of tuples containing: filename, content type, content.
         """
-        message = self._create_email(subject, attachment_content, attachment_name)
+        message = self._create_email(
+            subject, message_body_plain_text, message_body_html, attachments
+        )
         self._send_email(source_email_address, recipient_email_addresses, message)
         logger.debug(f"Logs sent to {recipient_email_addresses}")
 
     def _create_email(
         self,
         subject: str,
-        attachment_content: str,
-        attachment_name: str,
+        message_body_plain_text: str,
+        message_body_html: str | None = None,
+        attachments: list | None = None,
     ) -> MIMEMultipart:
-        """Create an email.
-
-        Args:
-            subject: The subject of the email.
-            attachment_content: The content of the email attachment.
-            attachment_name: The name of the email attachment.
-        """
         message = MIMEMultipart()
         message["Subject"] = subject
-        attachment_object = MIMEApplication(attachment_content)
-        attachment_object.add_header(
-            "Content-Disposition", "attachment", filename=attachment_name
-        )
-        message.attach(attachment_object)
+
+        message.attach(MIMEText(message_body_plain_text, "plain"))
+
+        if message_body_html:
+            message.attach(MIMEText(message_body_html, "html"))
+
+        if attachments:
+            for filename, content in attachments:
+                attachment = self._create_attachment(filename, content)
+                message.attach(attachment)
         return message
 
+    def _create_attachment(self, filename: str, content: StringIO) -> MIMEApplication:
+        attachment = MIMEApplication(content.read())
+        attachment.add_header("Content-Disposition", "attachment", filename=filename)
+        return attachment
+
     def _send_email(
         self,
         source_email_address: str,