From 71a82ea3ff3e212ce6670e2c29a2b28d6558d47d Mon Sep 17 00:00:00 2001 From: Eric Hanson Date: Fri, 29 Oct 2021 11:14:56 -0400 Subject: [PATCH] Add send DSS message to deposit function * Update deposit function to send message to DSS * Add arguments to deposit function * Rename package_files var to correspond with new name of function --- awd/deposit.py | 35 ++++++++++++++++++++++++------- tests/test_deposit.py | 49 +++++++++++++++++++++++++++++++++---------- 2 files changed, 65 insertions(+), 19 deletions(-) diff --git a/awd/deposit.py b/awd/deposit.py index 041111f..1add82f 100644 --- a/awd/deposit.py +++ b/awd/deposit.py @@ -3,15 +3,23 @@ from botocore.exceptions import ClientError -from awd import crossref, s3, wiley -from awd.s3 import S3 +from awd import crossref, s3, sqs, wiley logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) -def deposit(doi_spreadsheet_path, metadata_url, content_url, bucket): - s3_client = S3() +def deposit( + doi_spreadsheet_path, + metadata_url, + content_url, + bucket, + sqs_input_url, + sqs_output_url, + collection_handle, +): + s3_client = s3.S3() + sqs_client = sqs.SQS() dois = crossref.get_dois_from_spreadsheet(doi_spreadsheet_path) for doi in dois: crossref_work_record = crossref.get_work_record_from_doi(metadata_url, doi) @@ -25,19 +33,30 @@ def deposit(doi_spreadsheet_path, metadata_url, content_url, bucket): if wiley.is_valid_response(doi, wiley_response) is False: continue doi_file_name = doi.replace("/", "-") # 10.1002/term.3131 to 10.1002-term.3131 - package_files = s3.create_files_dict( + files_dict = s3.create_files_dict( doi_file_name, json.dumps(metadata), wiley_response.content ) try: - for file in package_files: + for file in files_dict: s3_client.put_file(file["file_content"], bucket, file["file_name"]) except ClientError as e: logger.error( f"Upload failed: {file['file_name']}, {e.response['Error']['Message']}" ) continue - # create dss message - # submit dss message + bitstream_s3_uri = f"s3://{bucket}/{doi_file_name}.pdf" + metadata_s3_uri = f"s3://{bucket}/{doi_file_name}.json" + dss_message_attributes = sqs.create_dss_message_attributes( + doi_file_name, "wiley", sqs_output_url + ) + dss_message_body = sqs.create_dss_message_body( + "DSpace", + collection_handle, + metadata_s3_uri, + f"{doi_file_name}.pdf", + bitstream_s3_uri, + ) + sqs_client.send(sqs_input_url, dss_message_attributes, dss_message_body) return "Submission process has completed" diff --git a/tests/test_deposit.py b/tests/test_deposit.py index 3143afb..6edf6cd 100644 --- a/tests/test_deposit.py +++ b/tests/test_deposit.py @@ -1,63 +1,90 @@ import logging +import boto3 +from moto import mock_sqs + from awd import deposit -from awd.s3 import S3 logger = logging.getLogger(__name__) -def test_deposit_success(web_mock, s3_mock, s3_client): - s3 = S3() +@mock_sqs +def test_deposit_success( + web_mock, s3_mock, s3_class, sqs_class, submission_message_body +): + sqs = boto3.resource("sqs", region_name="us-east-1") + sqs.create_queue(QueueName="mock-input-queue") response = deposit.deposit( "tests/fixtures/doi_success.csv", "http://example.com/works/", "http://example.com/doi/", "awd", + "https://queue.amazonaws.com/123456789012/mock-input-queue", + "https://queue.amazonaws.com/123456789012/mock-output-queue", + "123.4/5678", + ) + uploaded_metadata = s3_class.client.get_object( + Bucket="awd", Key="10.1002-term.3131.json" ) - uploaded_metadata = s3.client.get_object(Bucket="awd", Key="10.1002-term.3131.json") assert uploaded_metadata["ResponseMetadata"]["HTTPStatusCode"] == 200 - uploaded_bitstream = s3.client.get_object(Bucket="awd", Key="10.1002-term.3131.pdf") + uploaded_bitstream = s3_class.client.get_object( + Bucket="awd", Key="10.1002-term.3131.pdf" + ) assert uploaded_bitstream["ResponseMetadata"]["HTTPStatusCode"] == 200 + messages = sqs_class.receive( + "https://queue.amazonaws.com/123456789012/mock-input-queue" + ) + for message in messages: + assert message["Body"] == str(submission_message_body) assert response == "Submission process has completed" -def test_deposit_insufficient_metadata(caplog, web_mock, s3_mock, s3_client): +def test_deposit_insufficient_metadata(caplog, web_mock, s3_mock, s3_class): with caplog.at_level(logging.INFO): response = deposit.deposit( "tests/fixtures/doi_insufficient_metadata.csv", "http://example.com/works/", "http://example.com/doi/", "awd", + "mock-input-queue", + "mock-output-queue", + "123.4/5678", ) assert ( "Insufficient metadata for 10.1002/nome.tadata, missing title or URL" in caplog.text ) - assert "Contents" not in s3_client.client.list_objects(Bucket="awd") + assert "Contents" not in s3_class.client.list_objects(Bucket="awd") assert response == "Submission process has completed" -def test_deposit_pdf_unavailable(caplog, web_mock, s3_mock, s3_client): +def test_deposit_pdf_unavailable(caplog, web_mock, s3_mock, s3_class): with caplog.at_level(logging.INFO): response = deposit.deposit( "tests/fixtures/doi_pdf_unavailable.csv", "http://example.com/works/", "http://example.com/doi/", "awd", + "mock-input-queue", + "mock-output-queue", + "123.4/5678", ) assert "A PDF could not be retrieved for DOI: 10.1002/none.0000" in caplog.text - assert "Contents" not in s3_client.client.list_objects(Bucket="awd") + assert "Contents" not in s3_class.client.list_objects(Bucket="awd") assert response == "Submission process has completed" -def test_deposit_s3_upload_failed(caplog, web_mock, s3_mock, s3_client): +def test_deposit_s3_upload_failed(caplog, web_mock, s3_mock, s3_class): with caplog.at_level(logging.INFO): response = deposit.deposit( "tests/fixtures/doi_success.csv", "http://example.com/works/", "http://example.com/doi/", "not-a-bucket", + "mock-input-queue", + "mock-output-queue", + "123.4/5678", ) assert "Upload failed: 10.1002-term.3131.json" in caplog.text - assert "Contents" not in s3_client.client.list_objects(Bucket="awd") + assert "Contents" not in s3_class.client.list_objects(Bucket="awd") assert response == "Submission process has completed"