Merge pull request #192 from dbmi-bgm/kmp_submit_data_sans_upload_sbr2

Support for SubmitCGAP in cgap-portal (C4-302)

deploy/ini_files/any.ini

@@ -5,6 +5,7 @@ file_upload_bucket = elasticbeanstalk-${S3_BUCKET_ENV}-files
 file_wfout_bucket = elasticbeanstalk-${S3_BUCKET_ENV}-wfoutput
 blob_bucket = elasticbeanstalk-${S3_BUCKET_ENV}-blobs
 system_bucket = elasticbeanstalk-${S3_BUCKET_ENV}-system
+metadata_bundles_bucket = elasticbeanstalk-${S3_BUCKET_ENV}-metadata-bundles
 sentry_dsn = ${SENTRY_DSN}
 # blob_store_profile_name = encoded-4dn-files
 accession_factory = encoded.server_defaults.enc_accession

deploy/ini_files/cgap.ini

@@ -5,6 +5,7 @@ file_upload_bucket = elasticbeanstalk-fourfront-cgap-files
 file_wfout_bucket = elasticbeanstalk-fourfront-cgap-wfoutput
 blob_bucket = elasticbeanstalk-fourfront-cgap-blobs
 system_bucket = elasticbeanstalk-fourfront-cgap-system
+metadata_bundles_bucket = elasticbeanstalk-fourfront-cgap-metadata-bundles
 sentry_dsn = ${SENTRY_DSN}
 # blob_store_profile_name = encoded-4dn-files
 accession_factory = encoded.server_defaults.enc_accession

deploy/ini_files/cgapdev.ini

@@ -5,6 +5,7 @@ file_upload_bucket = elasticbeanstalk-fourfront-cgapdev-files
 file_wfout_bucket = elasticbeanstalk-fourfront-cgapdev-wfoutput
 blob_bucket = elasticbeanstalk-fourfront-cgapdev-blobs
 system_bucket = elasticbeanstalk-fourfront-cgapdev-system
+metadata_bundles_bucket = elasticbeanstalk-fourfront-cgapdev-metadata-bundles
 sentry_dsn = ${SENTRY_DSN}
 # blob_store_profile_name = encoded-4dn-files
 accession_factory = encoded.server_defaults.enc_accession

deploy/ini_files/cgaptest.ini

@@ -5,6 +5,7 @@ file_upload_bucket = elasticbeanstalk-fourfront-cgaptest-files
 file_wfout_bucket = elasticbeanstalk-fourfront-cgaptest-wfoutput
 blob_bucket = elasticbeanstalk-fourfront-cgaptest-blobs
 system_bucket = elasticbeanstalk-fourfront-cgaptest-system
+metadata_bundles_bucket = elasticbeanstalk-fourfront-cgaptest-metadata-bundles
 sentry_dsn = ${SENTRY_DSN}
 # blob_store_profile_name = encoded-4dn-files
 accession_factory = encoded.server_defaults.enc_accession

deploy/ini_files/cgapwolf.ini

@@ -5,6 +5,7 @@ file_upload_bucket = elasticbeanstalk-fourfront-cgapwolf-files
 file_wfout_bucket = elasticbeanstalk-fourfront-cgapwolf-wfoutput
 blob_bucket = elasticbeanstalk-fourfront-cgapwolf-blobs
 system_bucket = elasticbeanstalk-fourfront-cgapwolf-system
+metadata_bundles_bucket = elasticbeanstalk-fourfront-cgapwolf-metadata-bundles
 sentry_dsn = ${SENTRY_DSN}
 # blob_store_profile_name = encoded-4dn-files
 accession_factory = encoded.server_defaults.enc_accession

development.ini

@@ -7,6 +7,7 @@
 use = config:base.ini#app
 sqlalchemy.url = postgresql://postgres@localhost:5441/postgres?host=/tmp/snovault/pgdata
 blob_bucket = encoded-4dn-blobs
+metadata_bundles_bucket = elasticbeanstalk-fourfront-cgaplocal-dev-metadata-bundles
 load_test_only = true
 create_tables = true
 testing = true

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 # Note: Various modules refer to this system as "encoded", not "cgap-portal".
 name = "encoded"
-version = "3.0.0"
+version = "3.1.0"
 description = "Clinical Genomics Analysis Platform"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"
@@ -115,11 +115,11 @@ xlwt = "1.2.0"
 "zope.deprecation" = "4.4.0"
 "zope.interface" = "4.6.0"
 "zope.sqlalchemy" = "1.3"
-sentry-sdk = "^0.16.4"
+sentry-sdk = "^0.16.5"
 
 [tool.poetry.dev-dependencies]
 # PyCharm says boto3-stubs contains useful type hints
-boto3-stubs = ">=1.14.37.0"
+boto3-stubs = ">=1.14.55.2"
 coverage = ">=5.2"
 codacy-coverage = ">=1.3.11"
 coveralls = ">=2.1.1"
@@ -179,6 +179,9 @@ profiler = "encoded.commands.profiler:main"
 purge-item-type = "encoded.commands.purge_item_type:main"
 run-upgrade-on-inserts = "encoded.commands.run_upgrader_on_inserts:main"
 spreadsheet-to-json = "encoded.commands.spreadsheet_to_json:main"
+submission-test = "encoded.commands.submission_test:main"
+# Use the same-named script in SubmitCGAP instead.
+# submit-metadata-bundle = "encoded.commands.submit_metadata_bundle:main"
 update-inserts-from-server = "encoded.commands.update_inserts_from_server:main"
 verify-item = "encoded.commands.verify_item:main"
 

src/encoded/authentication.py

@@ -260,27 +260,6 @@ def get_token_info(self, token, request):
         return None
 
 
-# def get_jwt(request):
-#     token = None
-#     try:
-#         # ensure this is a jwt token not basic auth:
-#         auth_type = request.headers['Authorization'][:6]
-#         if auth_type.strip().lower() == 'bearer':
-#             token = request.headers['Authorization'][7:]
-#     except (ValueError, TypeError, KeyError):
-#         pass
-#
-#     if not token and request.method in ('GET', 'HEAD'):
-#         # Only grab this if is a GET request, not a transactional request to help mitigate CSRF attacks.
-#         # See: https://en.wikipedia.org/wiki/Cross-site_request_forgery#Cookie-to-header_token
-#         # The way our JS grabs and sticks JWT into Authorization header is somewhat analogous to above approach.
-#         # TODO: Ensure our `Access-Control-Allow-Origin` response headers are appropriate (more for CGAP).
-#         # TODO: Get a security audit done.
-#         token = request.cookies.get('jwtToken')
-#
-#     return token
-
-
 def get_jwt(request):
 
     token = None

src/encoded/commands/create_mapping_on_deploy.py

@@ -62,7 +62,8 @@
     'EvidenceDisPheno',
     'Page',
     'GeneAnnotationField',
-    'HiglassViewConfig'
+    'HiglassViewConfig',
+    'IngestionSubmission',
 ]
 
 ENV_HOTSEAT = 'fourfront-cgaphot'

src/encoded/commands/submission_test.py

@@ -0,0 +1,94 @@
+import io
+import json
+import os
+
+from dcicutils.misc_utils import VirtualApp
+from pyramid.paster import get_app
+from ..submit import digest_xls, xls_to_json, validate_all_items, post_and_patch_all_items
+from ..tests.data import DBMI_INSTITUTION, TEST_PROJECT, METADATA_BUNDLE_PATH
+
+
+TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "submission_test_data")
+
+TEST_FILE_TO_VALIDATE = os.path.join(TEST_DATA_DIR, "to_validate.json")
+TEST_FILE_TO_PATCH = os.path.join(TEST_DATA_DIR, "to_patch.json")
+TEST_FILE_TO_POST = os.path.join(TEST_DATA_DIR, "to_post.json")
+TEST_FILE_POST_OUTPUT = os.path.join(TEST_DATA_DIR, "post_output.json")
+TEST_FILE_PATCH_OUTPUT = os.path.join(TEST_DATA_DIR, "patch_output.json")
+TEST_FILE_TO_UPLOAD = os.path.join(TEST_DATA_DIR, "to_upload.json")
+
+with io.open(TEST_FILE_TO_VALIDATE, 'r') as fp:
+    TEST_DATA_TO_VALIDATE = json.load(fp)
+
+with io.open(TEST_FILE_TO_POST, 'r') as fp:
+    TEST_DATA_TO_POST = json.load(fp)
+
+with io.open(TEST_FILE_TO_PATCH, 'r') as fp:
+    TEST_DATA_TO_PATCH = json.load(fp)
+
+with io.open(TEST_FILE_POST_OUTPUT, 'r') as fp:
+    TEST_DATA_POST_OUTPUT = json.load(fp)
+
+with io.open(TEST_FILE_PATCH_OUTPUT, 'r') as fp:
+    TEST_DATA_PATCH_OUTPUT = json.load(fp)
+
+with io.open(TEST_FILE_TO_UPLOAD, 'r') as fp:
+    TEST_DATA_TO_UPLOAD = json.load(fp)
+
+
+def main():
+    """
+    This does a simple test of the data pipeline used for metadata bundle submission.
+
+    This does not test the submission endpoints, but DOES call server endpoints to
+    inquire about current state and based on that state to decide to post or patch the data.
+    (As such, the exact action of this script is dependent on the state of the db when it is run.)
+
+    The server called will be the one defined by the development environment.
+    As such, that means we won't be using this test as part of unit tests,
+    though perhaps variations of this will be adopted for that purpose.
+    """
+
+    app = get_app('development.ini', 'app')
+    environ = {'HTTP_ACCEPT': 'application/json', 'REMOTE_USER': 'TEST'}
+    virtualapp = VirtualApp(app, environ)
+    proj = virtualapp.get(TEST_PROJECT).json
+    inst = virtualapp.get(DBMI_INSTITUTION).json
+    rows = digest_xls(METADATA_BUNDLE_PATH)
+    json_data, passing = xls_to_json(rows, proj, inst)
+    print('JSON data (to validate):', json.dumps(json_data))
+    assert json_data == TEST_DATA_TO_VALIDATE
+    print('JSON data to validate matches contents of %s' % TEST_FILE_TO_VALIDATE)
+    final_json, validation_log, passing = validate_all_items(virtualapp, json_data)
+    print('Validation Log:\n'.join(validation_log))
+    print("Passing (after validation):", passing)
+    print("Final JSON (to post, after validation):", json.dumps(final_json, indent=4))
+    if final_json == TEST_DATA_TO_PATCH:
+        # NOTE: There are more possible intermediate states than just "it's all been done" and "none has been done",
+        #  but this simple script does not anticipate those and will just fail if one of those other states is in play.
+        #  -kmp 8-Sep-2020
+        posting = False
+        print("JSON data has already been posted. Light patching is expected.")
+        print("(To test posting at this point would require wiping the database or adjusting numerous items.)")
+        print('JSON data to patch matches contents of %s' % TEST_FILE_TO_PATCH)
+    else:
+        posting = True
+        assert final_json == TEST_DATA_TO_POST
+        print('JSON data to post matches contents of %s' % TEST_FILE_TO_POST)
+    output, passing, files = post_and_patch_all_items(virtualapp, final_json)
+    print('Post Output:\n', '\n'.join(output))
+    print('Passing (after post and patch):', passing)
+    if posting:
+        assert output == TEST_DATA_POST_OUTPUT
+        print('JSON data to post matches contents of %s' % TEST_FILE_POST_OUTPUT)
+    else:
+        assert output == TEST_DATA_PATCH_OUTPUT
+        print('JSON data to post matches contents of %s' % TEST_FILE_PATCH_OUTPUT)
+    print('Files:', json.dumps(files, indent=4))
+    assert files == TEST_DATA_TO_UPLOAD
+    print('JSON data to upload matches contents of %s' % TEST_FILE_TO_UPLOAD)
+    print("SUCCESS! All done.")
+
+
+if __name__ == '__main__':
+    main()

src/encoded/commands/submission_test_data/patch_output.json

@@ -0,0 +1,3 @@
+[
+    "file_fastq: 4 items patched successfully; 0 items not patched"
+]

src/encoded/commands/submission_test_data/post_output.json

@@ -0,0 +1,23 @@
+[
+    "Success - sample 3464467 posted",
+    "Success - sample 3464468 posted",
+    "Success - sample 3464469 posted",
+    "Success - individual 456 posted",
+    "Success - individual 789 posted",
+    "Success - individual 123 posted",
+    "Success - family 333 posted",
+    "file_fastq: 4 items posted successfully; 0 items not posted",
+    "sample: 3 items posted successfully; 0 items not posted",
+    "individual: 3 items posted successfully; 0 items not posted",
+    "family: 1 items posted successfully; 0 items not posted",
+    "sample_processing: 1 items posted successfully; 0 items not posted",
+    "report: 1 items posted successfully; 0 items not posted",
+    "case: 3 items posted successfully; 0 items not posted",
+    "file_fastq: 4 items patched successfully; 0 items not patched",
+    "sample: 3 items patched successfully; 0 items not patched",
+    "individual: 3 items patched successfully; 0 items not patched",
+    "family: 1 items patched successfully; 0 items not patched",
+    "sample_processing: 1 items patched successfully; 0 items not patched",
+    "report: 1 items patched successfully; 0 items not patched",
+    "case: 3 items patched successfully; 0 items not patched"
+]

src/encoded/commands/submission_test_data/to_patch.json

@@ -0,0 +1,47 @@
+{
+    "post": {},
+    "patch": {
+        "file_fastq": {
+            "/files-fastq/GAPFINDNNXMD/": {
+                "filename": "f1.fastq.gz",
+                "status": "uploading"
+            },
+            "/files-fastq/GAPFI8NPRCLA/": {
+                "filename": "f2.fastq.gz",
+                "status": "uploading"
+            },
+            "/files-fastq/GAPFIHLOD3W5/": {
+                "filename": "f3.fastq.gz",
+                "status": "uploading"
+            },
+            "/files-fastq/GAPFI558AY9P/": {
+                "filename": "f4.fastq.gz",
+                "status": "uploading"
+            }
+        },
+        "sample": {},
+        "individual": {},
+        "family": {},
+        "sample_processing": {},
+        "report": {},
+        "case": {}
+    },
+    "aliases": {
+        "hms-dbmi:f1.fastq.gz": "/files-fastq/GAPFINDNNXMD/",
+        "hms-dbmi:f2.fastq.gz": "/files-fastq/GAPFI8NPRCLA/",
+        "hms-dbmi:f3.fastq.gz": "/files-fastq/GAPFIHLOD3W5/",
+        "hms-dbmi:f4.fastq.gz": "/files-fastq/GAPFI558AY9P/",
+        "hms-dbmi:sample-3464467": "/samples/GAPSAOZBTKEV/",
+        "hms-dbmi:sample-3464468": "/samples/GAPSALGBXFHS/",
+        "hms-dbmi:sample-3464469": "/samples/GAPSA4YIRPGC/",
+        "hms-dbmi:individual-456": "/individuals/GAPIDXBYJESJ/",
+        "hms-dbmi:individual-789": "/individuals/GAPID16TYLTK/",
+        "hms-dbmi:individual-123": "/individuals/GAPIDF5WPST7/",
+        "hms-dbmi:family-456": "/families/GAPFA5SLA4GB/",
+        "hms-dbmi:analysis-55432": "/sample-processings/43a03760-4e11-41ae-81d9-a408acd29f16/",
+        "hms-dbmi:report-55432-3464467": "/reports/ea7490cb-d480-4ecf-b913-467fc24d7294/",
+        "hms-dbmi:case-55432-3464467": "/cases/GAPCA12VO85Q/",
+        "hms-dbmi:case-55432-3464468": "/cases/GAPCAWSV5574/",
+        "hms-dbmi:case-55432-3464469": "/cases/GAPCA2SA3Z84/"
+    }
+}