Changes based on end-to-end testing

* Adjust unit tests to better reflect actual AWS outputs
* Remove unnecessary logging config
* Adjust log level of messages for desired output
* Adjust SSM parameters based changes in DSS parameters
* Changes SES send_email function to take a single address (for a moira list)

* https://mitlibraries.atlassian.net/browse/DLSPP-134

awd/cli.py

@@ -12,13 +12,7 @@
 from awd.ses import SES
 from awd.sqs import SQS
 
-stream = io.StringIO()
 logger = logging.getLogger(__name__)
-logging.basicConfig(
-    format="%(asctime)s %(levelname)-8s %(message)s",
-    level=logging.ERROR,
-    handlers=[logging.StreamHandler(), logging.StreamHandler(stream)],
-)
 
 
 class Status(Enum):
@@ -41,7 +35,9 @@ def doi_to_be_retried(doi, doi_items):
     "Validate that a DOI should be retried based on its status in the database table."
     validation_status = False
     for doi_item in [
-        d for d in doi_items if d["doi"] == doi and d["status"] == "Failed, will retry"
+        d
+        for d in doi_items
+        if d["doi"] == doi and d["status"] == str(Status.FAILED.value)
     ]:
         validation_status = True
         logger.debug(f"{doi} will be retried.")
@@ -76,7 +72,6 @@ def doi_to_be_retried(doi, doi_items):
 @click.option(
     "--log_recipient_email",
     required=True,
-    multiple=True,
     default=config.LOG_RECIPIENT_EMAIL,
     help="The email address receiving the logs. Repeatable",
 )
@@ -89,12 +84,19 @@ def cli(
     log_source_email,
     log_recipient_email,
 ):
+    stream = io.StringIO()
+    logging.basicConfig(
+        format="%(asctime)s %(levelname)-8s %(message)s",
+        level=logging.INFO,
+        handlers=[logging.StreamHandler(), logging.StreamHandler(stream)],
+    )
     ctx.ensure_object(dict)
     ctx.obj["doi_table"] = doi_table
     ctx.obj["sqs_base_url"] = sqs_base_url
     ctx.obj["sqs_output_queue"] = sqs_output_queue
     ctx.obj["log_source_email"] = log_source_email
     ctx.obj["log_recipient_email"] = log_recipient_email
+    ctx.obj["stream"] = stream
 
 
 @cli.command()
@@ -145,6 +147,7 @@ def deposit(
     collection_handle,
 ):
     date = datetime.today().strftime("%m-%d-%Y %H:%M:%S")
+    stream = ctx.obj["stream"]
     s3_client = s3.S3()
     sqs_client = sqs.SQS()
     dynamodb_client = dynamodb.DynamoDB()
@@ -198,7 +201,7 @@ def deposit(
         bitstream_s3_uri = f"s3://{bucket}/{doi_file_name}.pdf"
         metadata_s3_uri = f"s3://{bucket}/{doi_file_name}.json"
         dss_message_attributes = sqs.create_dss_message_attributes(
-            doi_file_name, "wiley", ctx.obj["sqs_output_queue"]
+            doi, "wiley", ctx.obj["sqs_output_queue"]
         )
         dss_message_body = sqs.create_dss_message_body(
             "DSpace@MIT",
@@ -213,7 +216,7 @@ def deposit(
             dss_message_attributes,
             dss_message_body,
         )
-    logger.info("Submission process has completed")
+    logger.debug("Submission process has completed")
 
     ses_client = ses.SES()
     message = ses_client.create_email(
@@ -224,10 +227,10 @@ def deposit(
     try:
         ses_client.send_email(
             ctx.obj["log_source_email"],
-            list(ctx.obj["log_recipient_email"]),
+            ctx.obj["log_recipient_email"],
             message,
         )
-        logger.info(f"Logs sent to {list(ctx.obj['log_recipient_email'])}")
+        logger.debug(f"Logs sent to {ctx.obj['log_recipient_email']}")
     except ClientError as e:
         logger.error(f"Failed to send logs: {e.response['Error']['Message']}")
 
@@ -244,8 +247,8 @@ def listen(
     ctx,
     retry_threshold,
 ):
-
     date = datetime.today().strftime("%m-%d-%Y %H:%M:%S")
+    stream = ctx.obj["stream"]
     sqs = SQS()
     dynamodb_client = DynamoDB()
     try:
@@ -281,6 +284,7 @@ def listen(
                     ctx.obj["sqs_output_queue"],
                     message["ReceiptHandle"],
                 )
+
                 dynamodb_client.update_doi_item_status_in_database(
                     ctx.obj["doi_table"], doi, Status.SUCCESS.value
                 )
@@ -298,9 +302,9 @@ def listen(
     try:
         ses_client.send_email(
             ctx.obj["log_source_email"],
-            list(ctx.obj["log_recipient_email"]),
+            ctx.obj["log_recipient_email"],
             message,
         )
-        logger.info(f"Logs sent to {list(ctx.obj['log_recipient_email'])}")
+        logger.debug(f"Logs sent to {ctx.obj['log_recipient_email']}")
     except ClientError as e:
         logger.error(f"Failed to send logs: {e.response['Error']['Message']}")

awd/config.py

@@ -7,9 +7,8 @@
 DSS_SSM_PATH = os.getenv("DSS_SSM_PATH")
 WILEY_SSM_PATH = os.getenv("WILEY_SSM_PATH")
 
-logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
-logger.info("Configuring awd for current env: %s", ENV)
+logger.debug("Configuring awd for current env: %s", ENV)
 
 AWS_REGION_NAME = "us-east-1"
 
@@ -19,14 +18,13 @@
     DOI_TABLE = ssm.get_parameter_value(f"{WILEY_SSM_PATH}{ENV}/dynamodb_table_name")
     METADATA_URL = ssm.get_parameter_value(f"{WILEY_SSM_PATH}{ENV}/wiley_metadata_url")
     CONTENT_URL = ssm.get_parameter_value(f"{WILEY_SSM_PATH}{ENV}/wiley_content_url")
-    BUCKET = ssm.get_parameter_value(f"{DSS_SSM_PATH}{ENV}/wiley_submit_s3_bucket")
-    SQS_BASE_URL = ssm.get_parameter_value(f"{DSS_SSM_PATH}{ENV}/SQS_base_url")
-    SQS_INPUT_QUEUE = ssm.get_parameter_value(
-        f"{DSS_SSM_PATH}{ENV}/SQS_dss_input_queue"
-    )
+    BUCKET = ssm.get_parameter_value(f"{WILEY_SSM_PATH}{ENV}/wiley_submit_s3_bucket")
+    SQS_BASE_URL = ssm.get_parameter_value(f"{WILEY_SSM_PATH}{ENV}/SQS_base_url")
+    SQS_INPUT_QUEUE = ssm.get_parameter_value(f"{DSS_SSM_PATH}{ENV}/dss_input_queue")
     SQS_OUTPUT_QUEUE = ssm.get_parameter_value(
-        f"{DSS_SSM_PATH}{ENV}/SQS_dss_wiley_output_queue"
-    )
+        f"{DSS_SSM_PATH}{ENV}/dss_output_queues"
+    ).split(",")[1]
+
     COLLECTION_HANDLE = ssm.get_parameter_value(
         f"{WILEY_SSM_PATH}{ENV}/wiley_collection_handle"
     )

awd/dynamodb.py

@@ -21,7 +21,7 @@ def add_doi_item_to_database(self, doi_table, doi):
             TableName=doi_table,
             Item={
                 "doi": {"S": doi},
-                "status": {"S": "Processing"},
+                "status": {"S": "2"},
                 "attempts": {"S": "0"},
             },
         )

awd/ses.py

@@ -26,11 +26,11 @@ def create_email(
         message.attach(attachment_object)
         return message
 
-    def send_email(self, source_email, recipients, message):
-        """Send email via SES. Recipients parameter must be a list and not a str."""
+    def send_email(self, source_email, recipient, message):
+        """Send email via SES."""
         response = self.client.send_raw_email(
             Source=source_email,
-            Destinations=recipients,
+            Destinations=[recipient],
             RawMessage={
                 "Data": message.as_string(),
             },

tests/test_cli.py

@@ -3,7 +3,7 @@
 import boto3
 from moto import mock_dynamodb2, mock_ses, mock_sqs
 
-from awd.cli import cli, doi_to_be_added, doi_to_be_retried
+from awd.cli import Status, cli, doi_to_be_added, doi_to_be_retried
 
 logger = logging.getLogger(__name__)
 
@@ -21,13 +21,13 @@ def test_doi_to_be_added_false():
 
 
 def test_doi_to_be_retried_true():
-    doi_items = [{"doi": "111.1/111", "status": "Failed, will retry"}]
+    doi_items = [{"doi": "111.1/111", "status": str(Status.FAILED.value)}]
     validation_status = doi_to_be_retried("111.1/111", doi_items)
     assert validation_status is True
 
 
 def test_doi_to_be_retried_false():
-    doi_items = [{"doi": "111.1/111", "status": "Success"}]
+    doi_items = [{"doi": "111.1/111", "status": str(Status.SUCCESS.value)}]
     validation_status = doi_to_be_retried("111.1/111", doi_items)
     assert validation_status is False
 
@@ -298,15 +298,15 @@ def test_listen_success(
             TableName="test_dois",
             Item={
                 "doi": {"S": "111.1/1111"},
-                "status": {"S": "Processing"},
+                "status": {"S": str(Status.PROCESSING.value)},
                 "attempts": {"S": "1"},
             },
         )
         dynamodb.put_item(
             TableName="test_dois",
             Item={
                 "doi": {"S": "222.2/2222"},
-                "status": {"S": "Processing"},
+                "status": {"S": str(Status.PROCESSING.value)},
                 "attempts": {"S": "1"},
             },
         )

tests/test_dynamodb.py

@@ -34,15 +34,15 @@ def test_dynamodb_retrieve_doi_items_from_database(dynamodb_class):
         TableName="test_dois",
         Item={
             "doi": {"S": "111.1/1111"},
-            "status": {"S": "Failed, will retry"},
+            "status": {"S": str(Status.FAILED.value)},
             "attempts": {"S": "1"},
         },
     )
     dois = dynamodb_class.retrieve_doi_items_from_database("test_dois")
     assert dois == [
         {
             "doi": "111.1/1111",
-            "status": "Failed, will retry",
+            "status": str(Status.FAILED.value),
             "attempts": "1",
         }
     ]
@@ -63,7 +63,7 @@ def test_dynamodb_retry_threshold_exceeded_false(dynamodb_class):
         TableName="test_dois",
         Item={
             "doi": {"S": "111.1/1111"},
-            "status": {"S": "Failed, will retry"},
+            "status": {"S": str(Status.FAILED.value)},
             "attempts": {"S": "1"},
         },
     )
@@ -88,7 +88,7 @@ def test_dynamodb_retry_threshold_exceeded_true(dynamodb_class):
         TableName="test_dois",
         Item={
             "doi": {"S": "111.1/1111"},
-            "status": {"S": "Failed, will retry"},
+            "status": {"S": str(Status.FAILED.value)},
             "attempts": {"S": "10"},
         },
     )
@@ -113,7 +113,7 @@ def test_dynamodb_update_doi_item_attempts_in_database(dynamodb_class):
         TableName="test_dois",
         Item={
             "doi": {"S": "111.1/1111"},
-            "status": {"S": "Failed, will retry"},
+            "status": {"S": str(Status.FAILED.value)},
             "attempts": {"S": "1"},
         },
     )
@@ -124,7 +124,7 @@ def test_dynamodb_update_doi_item_attempts_in_database(dynamodb_class):
     assert existing_item["Item"] == {
         "attempts": {"S": "1"},
         "doi": {"S": "111.1/1111"},
-        "status": {"S": "Failed, will retry"},
+        "status": {"S": str(Status.FAILED.value)},
     }
     update_response = dynamodb_class.update_doi_item_attempts_in_database(
         "test_dois", "111.1/1111"
@@ -137,7 +137,7 @@ def test_dynamodb_update_doi_item_attempts_in_database(dynamodb_class):
     assert updated_item["Item"] == {
         "attempts": {"S": "2"},
         "doi": {"S": "111.1/1111"},
-        "status": {"S": "Failed, will retry"},
+        "status": {"S": str(Status.FAILED.value)},
     }
 
 
@@ -156,7 +156,7 @@ def test_dynamodb_update_doi_item_status_in_database(dynamodb_class):
         TableName="test_dois",
         Item={
             "doi": {"S": "111.1/1111"},
-            "status": {"S": "Failed, will retry"},
+            "status": {"S": str(Status.FAILED.value)},
             "attempts": {"S": "1"},
         },
     )
@@ -167,7 +167,7 @@ def test_dynamodb_update_doi_item_status_in_database(dynamodb_class):
     assert existing_item["Item"] == {
         "attempts": {"S": "1"},
         "doi": {"S": "111.1/1111"},
-        "status": {"S": "Failed, will retry"},
+        "status": {"S": str(Status.FAILED.value)},
     }
     update_response = dynamodb_class.update_doi_item_status_in_database(
         "test_dois", "111.1/1111", Status.PROCESSING.value
@@ -180,5 +180,5 @@ def test_dynamodb_update_doi_item_status_in_database(dynamodb_class):
     assert updated_item["Item"] == {
         "attempts": {"S": "1"},
         "doi": {"S": "111.1/1111"},
-        "status": {"S": "1"},
+        "status": {"S": str(Status.PROCESSING.value)},
     }

tests/test_ses.py

@@ -23,7 +23,7 @@ def test_ses_send_email():
     message = message = MIMEMultipart()
     response = ses.SES().send_email(
         "noreply@example.com",
-        ["test@example.com"],
+        "test@example.com",
         message,
     )
     assert response["ResponseMetadata"]["HTTPStatusCode"] == 200