PR updates

Author: ehanson8

dsaps/cli.py

@@ -13,6 +13,16 @@
 
 logger = structlog.get_logger()
 
+cwd = os.getcwd()
+
+
+def validate_path(ctx, param, value):
+    """Validates th formatting of The submitted path"""
+    if value[-1] == '/':
+        return value
+    else:
+        raise click.BadParameter('Include / at the end of the path.')
+
 
 @click.group(chain=True)
 @click.option('--url', envvar='DSPACE_URL', required=True,)
@@ -51,11 +61,13 @@ def main(ctx, url, email, password):
 
 @main.command()
 @click.option('-m', '--metadata-csv', required=True,
-              type=click.Path(exists=True),
+              type=click.Path(exists=True, file_okay=True, dir_okay=False),
               help='The path to the CSV file of metadata for the items.')
-@click.option('--field-map', required=True, type=click.Path(exists=True),
+@click.option('--field-map', required=True,
+              type=click.Path(exists=True, file_okay=True, dir_okay=False),
               help='The path to JSON field mapping file.')
 @click.option('-d', '--content-directory', required=True,
+              type=click.Path(exists=True, dir_okay=True, file_okay=False),
               help='The full path to the content, either a directory of files '
               'or a URL for the storage location.')
 @click.option('-t', '--file-type',
@@ -116,15 +128,19 @@ def newcollection(ctx, community_handle, collection_name):
 
 @main.command()
 @click.option('-m', '--metadata-csv', required=True,
+              type=click.Path(exists=True, file_okay=True, dir_okay=False),
               help='The path of the CSV file of metadata.')
-@click.option('-o', '--output-directory', default='', required=True,
+@click.option('-o', '--output-directory',
+              type=click.Path(exists=True, dir_okay=True, file_okay=False),
+              default=f'{cwd}/', callback=validate_path,
               help='The path of the output files, include / at the end of the '
               'path.')
 @click.option('-d', '--content-directory', required=True,
               help='The full path to the content, either a directory of files '
               'or a URL for the storage location.')
-@click.option('-t', '--file-type', required=True,
-              help='The file type to be uploaded.')
+@click.option('-t', '--file-type',
+              help='The file type to be uploaded, if limited to one file '
+              'type.', default='*')
 def reconcile(metadata_csv, output_directory, content_directory, file_type):
     """Runs a reconciliation of the specified files and metadata that produces
      reports of files with no metadata, metadata with no files, metadata

dsaps/helpers.py

@@ -13,11 +13,9 @@ def create_csv_from_list(list_name, output):
 
 
 def create_file_list(file_path, file_type):
-    """Creates a dict of file IDs and file paths."""
+    """Creates a list of file names."""
     files = glob.glob(f'{file_path}/**/*.{file_type}', recursive=True)
-    file_list = []
-    for file in files:
-        file_list.append(os.path.basename(file))
+    file_list = [os.path.basename(file) for file in files]
     return file_list
 
 
@@ -37,37 +35,32 @@ def create_metadata_id_list(metadata_csv):
     metadata_ids = []
     with open(metadata_csv) as csvfile:
         reader = csv.DictReader(csvfile)
-        for row in [r for r in reader if r['file_identifier'] != '']:
-            metadata_ids.append(row['file_identifier'])
+        metadata_ids = [row['file_identifier'] for row in reader
+                        if row['file_identifier'] != '']
     return metadata_ids
 
 
 def match_files_to_metadata(file_list, metadata_ids):
     """Creates a list of files matched to metadata records."""
-    file_matches = []
-    for file_id in file_list:
-        for metadata_id in [m for m in metadata_ids
-                            if file_id.startswith(m)]:
-            file_matches.append(file_id)
+    file_matches = [file_id for metadata_id in metadata_ids
+                    for file_id in file_list
+                    if file_id.startswith(metadata_id)]
     return file_matches
 
 
 def match_metadata_to_files(file_list, metadata_ids):
     """Creates a list of metadata records matched to files."""
-    metadata_matches = []
-    for metadata_id in metadata_ids:
-        for file_id in [f for f in file_list
-                        if f.startswith(metadata_id)]:
-            metadata_matches.append(metadata_id)
+    metadata_matches = [metadata_id for f in file_list for metadata_id in
+                        metadata_ids if f.startswith(metadata_id)]
     return metadata_matches
 
 
-def update_metadata_csv(metadata_csv, output_path, metadata_matches):
+def update_metadata_csv(metadata_csv, output_directory, metadata_matches):
     """Creates an updated CSV of metadata records with matching files."""
     with open(metadata_csv) as csvfile:
         reader = csv.DictReader(csvfile)
         upd_md_file_name = f'updated-{os.path.basename(metadata_csv)}'
-        with open(f'{output_path}{upd_md_file_name}', 'w') as updated_csv:
+        with open(f'{output_directory}{upd_md_file_name}', 'w') as updated_csv:
             writer = csv.DictWriter(updated_csv, fieldnames=reader.fieldnames)
             writer.writeheader()
             for row in reader:

tests/conftest.py

@@ -68,7 +68,7 @@ def runner():
 
 
 @pytest.fixture(autouse=True)
-def web_mock(input_dir):
+def web_mock():
     with requests_mock.Mocker() as m:
         cookies = {'JSESSIONID': '11111111'}
         m.post('mock://example.com/login', cookies=cookies)

tests/test_helpers.py

@@ -15,7 +15,7 @@ def test_create_csv_from_list(output_dir):
 
 
 def test_create_file_list(input_dir):
-    """Test create_file_dict function."""
+    """Test create_file_listt function."""
     file_list = helpers.create_file_list(input_dir, 'pdf')
     for file_id in ['test_02.pdf', 'test_01.pdf', 'best_01.pdf']:
         assert file_id in file_list
@@ -46,19 +46,18 @@ def test_create_metadata_id_list(input_dir):
 
 def test_match_files_to_metadata():
     """Test match_files_to_metadata function."""
-    file_dict = {'test_01': 'files/test_01.pdf'}
+    file_list = ['test_01.pdf']
     metadata_ids = ['test', 'tast']
-    file_matches = helpers.match_files_to_metadata(file_dict, metadata_ids)
+    file_matches = helpers.match_files_to_metadata(file_list, metadata_ids)
     assert len(file_matches) == 1
-    assert 'test_01' in file_matches
+    assert 'test_01.pdf' in file_matches
 
 
 def test_match_metadata_to_files():
     """Test match_metadata_to_files function."""
-    file_dict = {'test_01': 'files/test_01.pdf',
-                 'tast_01': 'files/tast_01.pdf'}
+    file_list = ['test_01.pdf', 'tast_01.pdf']
     metadata_ids = ['test']
-    file_matches = helpers.match_metadata_to_files(file_dict, metadata_ids)
+    file_matches = helpers.match_metadata_to_files(file_list, metadata_ids)
     assert len(file_matches) == 1
     assert 'test' in file_matches