PR updates

Author: ehanson8

.gitignore

@@ -180,5 +180,6 @@ local/*
 *.csv
 !tests/files/*.csv
 *.json
+!config/*.json
 createItemMetadataFromCSV_*
 *.txt

config/aspace_mapping.json

@@ -0,0 +1,17 @@
+{
+  "file_identifier": {
+    "csv_field_name": "file_identifier",
+    "language": null,
+    "delimiter": ""
+  },
+  "dc.title": {
+    "csv_field_name": "title",
+    "language": "en_US",
+    "delimiter": ""
+  },
+  "dc.relation.isversionof": {
+    "csv_field_name": "uri",
+    "language": null,
+    "delimiter": ""
+  }
+}

config/standard_mapping.json

@@ -0,0 +1,22 @@
+{
+  "file_identifier": {
+    "csv_field_name": "file_identifier",
+    "language": null,
+    "delimiter": ""
+  },
+  "dc.title": {
+    "csv_field_name": "title",
+    "language": "en_US",
+    "delimiter": ""
+  },
+  "dc.relation.isversionof": {
+    "csv_field_name": "uri",
+    "language": null,
+    "delimiter": ""
+  },
+  "dc.contributor.author": {
+    "csv_field_name": "authors",
+    "language": null,
+    "delimiter": "|"
+  }
+}

dsaps/cli.py

@@ -14,7 +14,7 @@
 logger = structlog.get_logger()
 
 
-@click.group()
+@click.group(chain=True)
 @click.option('--url', envvar='DSPACE_URL')
 @click.option('-e', '--email', envvar='TEST_EMAIL',
               help='The email of the user for authentication.')
@@ -50,9 +50,6 @@ def main(ctx, url, email, password):
 
 
 @main.command()
-@click.option('-c', '--collection-handle', required=True,
-              help='The handle of the collection to which items are being '
-              'added.')
 @click.option('-m', '--metadata-csv', required=True,
               help='The full path to the CSV file of metadata for the items.')
 @click.option('--field-map', required=True,
@@ -66,64 +63,50 @@ def main(ctx, url, email, password):
 @click.option('-r', '--ingest-report', is_flag=True,
               help='Create ingest report for updating other systems.')
 @click.pass_context
-def additems(ctx, collection_handle, metadata_csv, field_map,
-             directory, file_type, ingest_report):
+def additems(ctx, metadata_csv, field_map, directory, file_type,
+             ingest_report):
     client = ctx.obj['client']
     start_time = ctx.obj['start_time']
+    collection_uuid = ctx.obj['collection_uuid']
     with open(metadata_csv, 'r') as csvfile, open(field_map, 'r') as jsonfile:
         metadata = csv.DictReader(csvfile)
         mapping = json.load(jsonfile)
         collection = Collection.from_csv(metadata, mapping)
     for item in collection.items:
         item.bitstreams_from_directory(directory, file_type)
-    collection_uuid = client.get_id_from_handle(collection_handle)
-    collection.handle = collection_handle
     collection.uuid = collection_uuid
-    collection.post_items(client)
+    items = collection.post_items(client)
+    if ingest_report:
+        report_name = metadata_csv.replace('.csv', '-ingest.csv')
+        helpers.create_ingest_report(items, report_name)
     helpers.elapsed_time(start_time, 'Total runtime:')
 
-#
-# @main.command()
-# @click.option('-c', '--comm_handle', prompt='Enter the community handle',
-#               help='The handle of the community in which to create the ,'
-#               'collection.')
-# @click.option('-n', '--coll_name', prompt='Enter the name of the collection',
-#               help='The name of the collection to be created.')
-# @click.option('-m', '--metadata_csv', prompt='Enter the metadata CSV file',
-#               help='The path of the CSV file of metadata.')
-# @click.option('-f', '--file_path', prompt='Enter the path',
-#               help='The path of the content, a URL or local drive path.')
-# @click.option('-t', '--file_type', prompt='Enter the file type',
-#               help='The file type to be uploaded.')
-# @click.option('-i', '--ingest_type', prompt='Enter the type of ingest',
-#               help='The type of ingest to perform: local, remote.',
-#               type=click.Choice(['local', 'remote']), default='remote')
-# @click.option('-r', '--ingest_report', prompt='Create an ingest report?',
-#               help='Create ingest report for updating other systems',
-#               default=False)
-# @click.option('-u', '--multiple_terms', prompt='Method of separating terms?',
-#               help='The way multiple terms are separated in the metadata CSV.',
-#               type=click.Choice(['delimited', 'num_columns']),
-#               default='delimited')
-# @click.pass_context
-# def newcoll(ctx, comm_handle, coll_name, metadata_csv, file_path, file_type,
-#             ingest_type, ingest_report, multiple_terms):
-#     client = ctx.obj['client']
-#     start_time = ctx.obj['start_time']
-#     ingest_data = {}
-#     json_metadata = metadata.create_json_metadata(metadata_csv, multiple_terms)
-#     items = workflows.populate_new_coll(client, comm_handle, coll_name,
-#                                         ingest_type, file_path, file_type,
-#                                         json_metadata, ingest_report,
-#                                         ingest_data)
-#     for item in items:
-#         logger.info(f'Item posted: {item}')
-#     if ingest_report == 'True':
-#         report_name = metadata_csv.replace('.csv', '-ingest.csv')
-#         helpers.create_ingest_report(ingest_data, report_name)
-#     helpers.elapsed_time(start_time, 'Total runtime:')
-#
-#
+
+@main.command()
+@click.option('-c', '--collection-handle', required=True,
+              help='The handle of the collection to which items are being '
+              'added.')
+@click.pass_context
+def existingcollection(ctx, collection_handle):
+    client = ctx.obj['client']
+    collection_uuid = client.get_id_from_handle(collection_handle)
+    ctx.obj['collection_uuid'] = collection_uuid
+
+
+@main.command()
+@click.option('-c', '--community-handle', required=True,
+              help='The handle of the community in which to create the ,'
+              'collection.')
+@click.option('-n', '--collection-name', required=True,
+              help='The name of the collection to be created.')
+@click.pass_context
+def newcollection(ctx, community_handle, collection_name):
+    client = ctx.obj['client']
+    collection_uuid = client.post_coll_to_comm(community_handle,
+                                               collection_name)
+    ctx.obj['collection_uuid'] = collection_uuid
+
+
 # @main.command()
 # @click.option('-m', '--metadata_csv', prompt='Enter the metadata CSV file',
 #               help='The path of the CSV file of metadata.')

dsaps/helpers.py

@@ -1,19 +1,15 @@
-import collections
 import csv
 import datetime
 import glob
 import os
-import requests
 import time
 
-from lxml import html
 import structlog
 
 
 logger = structlog.get_logger()
 
 
-
 def create_csv_from_list(list_name, output):
     """Creates CSV file from list content."""
     with open(f'{output}.csv', 'w') as csvfile:
@@ -25,24 +21,23 @@ def create_csv_from_list(list_name, output):
 
 def create_file_dict(file_path, file_type):
     """Creates a dict of file IDs and file paths."""
-    if file_path.startswith('http'):
-        file_dict = build_file_dict_remote(file_path, file_type, {})
-    else:
-        files = glob.glob(f'{file_path}/**/*.{file_type}', recursive=True)
-        file_dict = {}
-        for file in files:
-            file_name = os.path.splitext(os.path.basename(file))[0]
-            file_dict[file_name] = file
+    files = glob.glob(f'{file_path}/**/*.{file_type}', recursive=True)
+    file_dict = {}
+    for file in files:
+        file_name = os.path.splitext(os.path.basename(file))[0]
+        file_dict[file_name] = file
     return file_dict
 
 
-def create_ingest_report(ingest_data, file_name):
-    """Creates ingest report of handles and DOS links."""
-    with open(f'{file_name}.csv', 'w') as writecsv:
+def create_ingest_report(items, file_name):
+    """Creates ingest report of other systems' identifiers with a newly created
+     DSpace handle."""
+    with open(f'{file_name}', 'w') as writecsv:
         writer = csv.writer(writecsv)
         writer.writerow(['uri'] + ['link'])
-        for uri, handle in ingest_data.items():
-            writer.writerow([uri] + [f'https://hdl.handle.net/{handle}'])
+        for item in items:
+            writer.writerow([item.source_system_identifier]
+                            + [f'https://hdl.handle.net/{item.handle}'])
 
 
 def create_metadata_id_list(metadata_csv):
@@ -81,22 +76,6 @@ def match_metadata_to_files(file_dict, metadata_ids):
     return metadata_matches
 
 
-def select_bitstreams(ingest_type, file_dict, file_identifier):
-    """Select the appropriate bitstreams for posting to an item."""
-    sel_bitstreams = []
-    file_dict = collections.OrderedDict(sorted(file_dict.items()))
-    for k in [e for e in file_dict if e.startswith(file_identifier)]:
-        pass
-    for bitstream_id in [k for k, v in file_dict.items()
-                         if k.startswith(file_identifier)]:
-        if ingest_type == 'local':
-            data = open(file_dict[bitstream_id], 'rb')
-        elif ingest_type == 'remote':
-            data = requests.get(file_dict[bitstream_id]).content
-        sel_bitstreams.append(data)
-    return sel_bitstreams
-
-
 def update_metadata_csv(metadata_csv, output_path, metadata_matches):
     """Creates an updated CSV of metadata records with matching files."""
     with open(metadata_csv) as csvfile:

dsaps/models.py

@@ -2,10 +2,10 @@
 import glob
 import operator
 import os
-import structlog
 
 import attr
 import requests
+import structlog
 
 Field = partial(attr.ib, default=None)
 Group = partial(attr.ib, default=[])
@@ -111,10 +111,11 @@ def post_coll_to_comm(self, comm_handle, coll_name):
     def post_item_to_collection(self, collection_id, item):
         """Posts item to a specified collection and returns the item ID."""
         endpoint = f'{self.url}/collections/{collection_id}/items'
-        post_response = requests.post(endpoint, headers=self.header,
-                                      cookies=self.cookies,
-                                      json=attr.asdict(item)['metadata']).json()
+        post_response = requests.post(
+            endpoint, headers=self.header, cookies=self.cookies,
+            json={'metadata': attr.asdict(item)['metadata']}).json()
         item_id = post_response['uuid']
+        item.handle = post_response['handle']
         return item_id
 
     def _pop_inst(self, class_type, rec_obj):
@@ -155,14 +156,17 @@ class Collection(BaseRecord):
     def post_items(self, client):
         for item in self.items:
             item_id = client.post_item_to_collection(self.uuid, item)
+            item.uuid = item_id
+            logger.info(f'Item posted: {item_id}')
             for bitstream in item.bitstreams:
-                client.post_bitstream(item_id, bitstream)
-                yield item, bitstream
+                bitstream_id = client.post_bitstream(item_id, bitstream)
+                logger.info(f'Bitstream posted: {bitstream_id}')
+            yield item
 
     @classmethod
     def from_csv(cls, csv_reader, field_map):
         items = [
-            Item.metadata_from_row(row, field_map) for row in csv_reader
+            Item.from_row(row, field_map) for row in csv_reader
             ]
         return cls(items=items)
 
@@ -176,23 +180,33 @@ class Community(BaseRecord):
 class Item(BaseRecord):
     metadata = Group()
     bitstreams = Group()
+    file_identifier = Field()
+    source_system_identifier = Field()
 
     def bitstreams_from_directory(self, directory, file_type='*'):
-        file_identifier = [m.value for m in self.metadata if
-                           m.key == 'file_identifier'][0]
-        file_list = glob.iglob(
-            f'{directory}/**/{file_identifier}*.{file_type}', recursive=True
+        files = glob.iglob(
+            f'{directory}/**/{self.file_identifier}*.{file_type}',
+            recursive=True
             )
+        file_list = []
+        for file in files:
+            file_list.append(file)
+        file_list.sort()
         self.bitstreams = [
             Bitstream(name=os.path.basename(f),
                       file_path=f) for f in file_list
             ]
 
     @classmethod
-    def metadata_from_row(cls, row, field_map):
+    def from_row(cls, row, field_map):
         metadata = []
         for f in field_map:
             field = row[field_map[f]['csv_field_name']]
+            if f == 'file_identifier':
+                file_identifier = field
+                continue  # file_identifier is not included in DSpace metadata
+            if f == 'dc.relation.isversionof':
+                source_system_identifier = field
             delimiter = field_map[f]['delimiter']
             language = field_map[f]['language']
             if delimiter:
@@ -204,7 +218,8 @@ def metadata_from_row(cls, row, field_map):
                 metadata.append(
                     MetadataEntry(key=f, value=field, language=language)
                     )
-        return cls(metadata=metadata)
+        return cls(metadata=metadata, file_identifier=file_identifier,
+                   source_system_identifier=source_system_identifier)
 
 
 @attr.s