FF-418 #comment more file operation tests

README.md

@@ -6,7 +6,7 @@
 [![Code Quality](https://api.codacy.com/project/badge/Grade/a4d521b4dd9c49058304606714528538)](https://www.codacy.com/app/jeremy_7/Submit4DN)
 [![PyPI version](https://badge.fury.io/py/Submit4DN.svg)](https://badge.fury.io/py/Submit4DN)
 
-The Submit4DN package is written by the [4DN Data Coordination and Integration Center](http://dcic.4dnucleome.org/) for data submitters from the 4DN Network. Please [contact us](mailto:4DN.DCIC.support@4dn-dcic.atlassian.net) to get access to the system, or if you have any questions or suggestions.  Detailed documentation on data submission can be found [at this link](https://docs.google.com/document/d/1Xh4GxapJxWXCbCaSqKwUd9a2wTiXmfQByzP0P8q5rnE/edit?usp=sharing)
+The Submit4DN package is written by the [4DN Data Coordination and Integration Center](http://dcic.4dnucleome.org/) for data submitters from the 4DN Network. Please [contact us](mailto:support@4dnucleome.org) to get access to the system, or if you have any questions or suggestions.  Detailed documentation on data submission can be found [at this link](https://docs.google.com/document/d/1Xh4GxapJxWXCbCaSqKwUd9a2wTiXmfQByzP0P8q5rnE/edit?usp=sharing)
 
 ## Installing the package
 
@@ -47,7 +47,7 @@ Once installed then follow the directions below:
 
 ## Connection
 To be able to use the provided tools, you need to have a secure key to access the REST application.
-If you do not have a secure key, please contact [4DN Data Wranglers](mailto:4DN.DCIC.support@hms-dbmi.atlassian.net)
+If you do not have a secure key, please contact [4DN Data Wranglers](mailto:support@4dnucleome.org)
 to get an account and to learn how to generate a key. Place your key in a json file in the following format.
 
     {

tests/conftest.py

@@ -253,6 +253,10 @@ def returned_vendor_schema():
     data = {"title":"Vendor","description":"Schema for submitting an originating lab or vendor.","id":"/profiles/vendor.json","$schema":"http://json-schema.org/draft-04/schema#","type":"object","required":["title"],"identifyingProperties":["uuid","name"],"additionalProperties":False,"mixinProperties":[{"$ref":"mixins.json#/schema_version"},{"$ref":"mixins.json#/uuid"},{"$ref":"mixins.json#/status"},{"$ref":"mixins.json#/notes"},{"$ref":"mixins.json#/submitted"},{"$ref":"mixins.json#/attribution"},{"$ref":"mixins.json#/aliases"}],"properties":{"aliases":{"type":"array","default":[],"uniqueItems":True,"title":"Lab aliases","description":"Lab specific identifiers to reference an object.","items":{"comment":"Current convention is colon separated lab name and lab identifier. (e.g. john-doe:42).","pattern":"^\\S+:\\S+","uniqueKey":"alias","title":"Lab alias","description":"A lab specific identifier to reference an object.","type":"string"}},"award":{"comment":"See award.json for list of available identifiers.","title":"Grant","description":"Grant associated with the submission.","linkTo":"Award","type":"string"},"lab":{"description":"Lab associated with the submission.","linkSubmitsFor":True,"title":"Lab","comment":"See lab.json for list of available identifiers.","linkTo":"Lab","type":"string"},"date_created":{"anyOf":[{"format":"date-time"},{"format":"date"}],"serverDefault":"now","readonly":True,"type":"string","comment":"Do not submit, value is assigned by the server. The date the object is created.","title":"Date created","rdfs:subPropertyOf":"dc:created","permission":"import_items"},"submitted_by":{"serverDefault":"userid","readonly":True,"type":"string","comment":"Do not submit, value is assigned by the server. The user that created the object.","linkTo":"User","title":"Submitted by","rdfs:subPropertyOf":"dc:creator","permission":"import_items"},"notes":{"elasticsearch_mapping_index_type":{"title":"Field mapping index type","description":"Defines one of three types of indexing available","type":"string","enum":["analyzed","not_analyzed","no"],"default":"analyzed"},"description":"DCIC internal notes.","type":"string","title":"Notes"},"status":{"readonly":True,"default":"in review by lab","title":"Status","type":"string","enum":["released","current","revoked","deleted","replaced","in review by lab","in review by project","released to project"],"permission":"import_items"},"uuid":{"serverDefault":"uuid4","readonly":True,"requestMethod":"POST","type":"string","title":"UUID","format":"uuid","permission":"import_items"},"schema_version":{"default":"1","pattern":"^\\d+(\\.\\d+)*$","requestMethod":[],"title":"Schema Version","comment":"Do not submit, value is assigned by the server. The version of the JSON schema that the server uses to validate the object. Schema version indicates generation of schema used to save version to to enable upgrade steps to work. Individual schemas should set the default.","type":"string"},"description":{"title":"Description","description":"A plain text description of the source.","type":"string","default":""},"title":{"title":"Name","description":"The complete name of the originating lab or vendor. ","type":"string"},"name":{"uniqueKey":True,"type":"string","description":"DON'T SUBMIT, auto-generated, use for referencing vendors in other sheets.","pattern":"^[a-z0-9\\-]+$"},"url":{"title":"URL","description":"An external resource with additional information about the source.","type":"string","format":"uri"},"@type":{"calculatedProperty":True,"title":"Type","type":"array","items":{"type":"string"}},"@id":{"calculatedProperty":True,"title":"ID","type":"string"}},"boost_values":{"name":1,"title":1},"@type":["JSONSchema"]}
     return MockedResponse(data, 200)
 
+@pytest.fixture
+def returned_experiment_set_schema():
+    data = {"title":"Experiment set","description":"Schema for submitting metadata for an experiment set.","id":"/profiles/experiment_set.json","$schema":"http://json-schema.org/draft-04/schema#","type":"object","required":["award","lab"],"identifyingProperties":["uuid","aliases"],"additionalProperties":False,"mixinProperties":[{"$ref":"mixins.json#/schema_version"},{"$ref":"mixins.json#/accession"},{"$ref":"mixins.json#/uuid"},{"$ref":"mixins.json#/aliases"},{"$ref":"mixins.json#/status"},{"$ref":"mixins.json#/attribution"},{"$ref":"mixins.json#/submitted"},{"$ref":"mixins.json#/notes"},{"$ref":"mixins.json#/documents"}],"properties":{"documents":{"type":"array","title":"Documents","items":{"type":"string","linkTo":"Document","comment":"See document.json for available identifiers.","title":"Document","description":"A document that provides additional information (not data file)."},"default":[],"description":"Documents that provide additional information (not data file).","uniqueItems":True},"notes":{"type":"string","elasticsearch_mapping_index_type":{"type":"string","default":"analyzed","enum":["analyzed","not_analyzed","no"],"title":"Field mapping index type","description":"Defines one of three types of indexing available"},"title":"Notes","description":"DCIC internal notes."},"submitted_by":{"type":"string","linkTo":"User","comment":"Do not submit, value is assigned by the server. The user that created the object.","title":"Submitted by","rdfs:subPropertyOf":"dc:creator","readonly":True,"serverDefault":"userid","permission":"import_items"},"date_created":{"type":"string","serverDefault":"now","anyOf":[{"format":"date-time"},{"format":"date"}],"comment":"Do not submit, value is assigned by the server. The date the object is created.","title":"Date created","rdfs:subPropertyOf":"dc:created","readonly":True,"permission":"import_items"},"lab":{"type":"string","linkTo":"Lab","comment":"See lab.json for list of available identifiers.","title":"Lab","description":"Lab associated with the submission.","linkSubmitsFor":True},"award":{"type":"string","linkTo":"Award","comment":"See award.json for list of available identifiers.","title":"Grant","description":"Grant associated with the submission."},"status":{"type":"string","readonly":True,"title":"Status","enum":["released","current","revoked","deleted","replaced","in review by lab","in review by project","released to project"],"default":"in review by lab","permission":"import_items"},"aliases":{"type":"array","title":"Lab aliases","items":{"type":"string","pattern":"^\\S+:\\S+","comment":"Current convention is colon separated lab name and lab identifier. (e.g. john-doe:42).","title":"Lab alias","description":"A lab specific identifier to reference an object.","uniqueKey":"alias"},"default":[],"description":"Lab specific identifiers to reference an object.","uniqueItems":True},"uuid":{"type":"string","readonly":True,"title":"UUID","serverDefault":"uuid4","requestMethod":"POST","permission":"import_items","format":"uuid"},"accession":{"type":"string","accessionType":"ES","readonly":True,"title":"Accession","description":"A unique identifier to be used to reference the object.","serverDefault":"accession","permission":"import_items","comment":"Only admins are allowed to set or update this value.","format":"accession"},"alternate_accessions":{"type":"array","default":[],"description":"Accessions previously assigned to objects that have been merged with this object.","title":"Alternate accessions","items":{"type":"string","comment":"Only admins are allowed to set or update this value.","title":"Alternate Accession","description":"An accession previously assigned to an object that has been merged with this object.","permission":"import_items","format":"accession"}},"schema_version":{"type":"string","pattern":"^\\d+(\\.\\d+)*$","hidden comment":"Bump the default in the subclasses.","comment":"Do not submit, value is assigned by the server. The version of the JSON schema that the server uses to validate the object. Schema version indicates generation of schema used to save version to to enable upgrade steps to work. Individual schemas should set the default.","title":"Schema Version","requestMethod":[]},"experiments_in_set":{"type":"array","title":"Set of experiments","exclude_from":"submit4dn","default":[],"description":"List of experiments to be associatedas a set.","uniqueItems":True,"items":{"title":"Experiment","comment":"use accessions for identifiers.","type":"string","linkTo":"Experiment"}},"experimentset_type":{"type":"string","enum":["custom"],"title":"Experiment Set type","description":"The categorization of the set of experiments."},"description":{"type":"string","default":"","title":"Description","description":"A description of why experiments are part of the set."},"@type":{"type":"array","calculatedProperty":True,"title":"Type","items":{"type":"string"}},"@id":{"type":"string","calculatedProperty":True,"title":"ID"}},"facets":{"experimentset_type":{"title":"Experiment set type"},"experiments_in_set.award.project":{"title":"Project"},"experiments_in_set.biosample.biosource.individual.organism.name":{"title":"Organism"},"experiments_in_set.biosample.biosource.biosource_type":{"title":"Biosource type"},"experiments_in_set.biosample.biosource_summary":{"title":"Biosource"},"experiments_in_set.digestion_enzyme.name":{"title":"Enzyme"},"experiments_in_set.biosample.modifications_summary":{"title":"Modifications"},"experiments_in_set.biosample.treatments_summary":{"title":"Treatments"},"experiments_in_set.lab.title":{"title":"Lab"}},"columns":{"accession":{"title":"Accession"},"experimentset_type":{"title":"Experiment set type"},"description":{"title":"Description"},"experiments_in_set":{"title":"Experiments"}},"@type":["JSONSchema"]}
+    return MockedResponse(data, 200)
 
 @pytest.fixture
 def returned_vendor_items():

tests/test_get_field_info.py

@@ -104,9 +104,28 @@ def test_get_uploadable_fields_mock(connection, mocker, returned_vendor_schema):
             assert field.enum is not None
 
 
+def xls_to_list(xls_file, sheet):
+    from operator import itemgetter
+    import xlrd
+    return_list = []
+    wb = xlrd.open_workbook(xls_file)
+    read_sheet = wb.sheet_by_name(sheet)
+    cols = read_sheet.ncols
+    rows = read_sheet.nrows
+    for row_idx in range(rows):
+        row_val = []
+        for col_idx in range(cols):
+            cell_value = str(read_sheet.cell(row_idx, col_idx))
+
+            row_val.append(cell_value)
+        return_list.append(row_val)
+    return return_list.sort(key=itemgetter(1))
+
+
 @pytest.mark.file_operation
-def test_create_xls(connection, mocker, returned_vendor_schema):
-    xls_file = "./tests/data_files/Vendor_gfi_test.xls"
+def test_create_xls_vendor(connection, mocker, returned_vendor_schema):
+    xls_file = "./tests/data_files/GFI_test_vendor.xls"
+    xls_ref_file = "./tests/data_files/GFI_test_vendor_reference.xls"
     import os
     try:
         os.remove(xls_file)
@@ -116,7 +135,28 @@ def test_create_xls(connection, mocker, returned_vendor_schema):
         field_dict = gfi.get_uploadable_fields(connection, ['Vendor'])
         gfi.create_xls(field_dict, xls_file)
         assert os.path.isfile(xls_file)
+        assert xls_to_list(xls_file, "Vendor") == xls_to_list(xls_ref_file, "Vendor")
+    # try:
+    #     os.remove(xls_file)
+    # except OSError:
+    #     pass
+
+
+@pytest.mark.file_operation
+def test_create_xls_experiment_set(connection, mocker, returned_experiment_set_schema):
+    xls_file = "./tests/data_files/GFI_test_Experiment_Set.xls"
+    xls_ref_file = "./tests/data_files/GFI_test_Experiment_Set_reference.xls"
+    import os
     try:
         os.remove(xls_file)
     except OSError:
         pass
+    with mocker.patch('wranglertools.fdnDCIC.requests.get', return_value=returned_experiment_set_schema):
+        field_dict = gfi.get_uploadable_fields(connection, ['ExperimentSet'], True, True, True)
+        gfi.create_xls(field_dict, xls_file)
+        assert os.path.isfile(xls_file)
+        assert xls_to_list(xls_file, "ExperimentSet") == xls_to_list(xls_ref_file, "ExperimentSet")
+    # try:
+    #     os.remove(xls_file)
+    # except OSError:
+    #     pass