Skip to content

Commit 59550f4

Browse files
hakbaileyehanson8
hakbailey
authored and
ehanson8
committed
Restructure code examples
1 parent 6e35231 commit 59550f4

File tree

13 files changed

+514
-343
lines changed

13 files changed

+514
-343
lines changed

‎Pipfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ verify_ssl = true
66
[dev-packages]
77
pytest = "*"
88
requests-mock = "*"
9+
coverage = "*"
910

1011
[packages]
1112
requests = "*"

‎Pipfile.lock‎

Lines changed: 59 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎dsaps/cli.py‎

Lines changed: 88 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,24 @@
1+
import csv
12
import datetime
3+
import json
24
import logging
35
import os
46
import time
57

68
import click
79
import structlog
810

9-
from dsaps import helpers, metadata, models, workflows
11+
from dsaps.models import Client, Collection
12+
from dsaps import helpers
1013

1114
logger = structlog.get_logger()
1215

1316

1417
@click.group()
1518
@click.option('--url', envvar='DSPACE_URL')
16-
@click.option('-e', '--email', prompt='Enter email', envvar='TEST_EMAIL',
19+
@click.option('-e', '--email', envvar='TEST_EMAIL',
1720
help='The email of the user for authentication.')
18-
@click.option('-p', '--password', prompt='Enter password', envvar='TEST_PASS',
21+
@click.option('-p', '--password', envvar='TEST_PASS',
1922
hide_input=True, help='The password for authentication.')
2023
@click.pass_context
2124
def main(ctx, url, email, password):
@@ -38,7 +41,7 @@ def main(ctx, url, email, password):
3841
'w')],
3942
level=logging.INFO)
4043
logger.info('Application start')
41-
client = models.Client(url)
44+
client = Client(url)
4245
client.authenticate(email, password)
4346
start_time = time.time()
4447
ctx.obj['client'] = client
@@ -47,95 +50,94 @@ def main(ctx, url, email, password):
4750

4851

4952
@main.command()
50-
@click.option('-c', '--coll_handle', prompt='Enter the collection handle',
53+
@click.option('-c', '--collection-handle', required=True,
5154
help='The handle of the collection to which items are being '
5255
'added.')
53-
@click.option('-m', '--metadata_csv', prompt='Enter the metadata CSV file',
54-
help='The path of the CSV file of metadata.')
55-
@click.option('-f', '--file_path', prompt='Enter the path',
56-
help='The path of the content, a URL or local drive path.')
57-
@click.option('-t', '--file_type', prompt='Enter the file type',
58-
help='The file type to be uploaded.')
59-
@click.option('-i', '--ingest_type', prompt='Enter the type of ingest',
60-
help='The type of ingest to perform: local, remote.',
61-
type=click.Choice(['local', 'remote']), default='remote')
62-
@click.option('-r', '--ingest_report', prompt='Create an ingest report?',
63-
help='Create ingest report for updating other systems.',
64-
default=False)
65-
@click.option('-u', '--multiple_terms', prompt='Method of separating terms?',
66-
help='The way multiple terms are separated in the metadata CSV.',
67-
type=click.Choice(['delimited', 'num_columns']),
68-
default='delimited')
56+
@click.option('-m', '--metadata-csv', required=True,
57+
help='The full path to the CSV file of metadata for the items.')
58+
@click.option('--field-map', required=True,
59+
help='Path to JSON field mapping file')
60+
@click.option('-d', '--directory', required=True,
61+
help='The full path to the content, either a directory of files '
62+
'or a URL for the storage location.')
63+
@click.option('-t', '--file-type',
64+
help='The file type to be uploaded, if limited to one file '
65+
'type.', default='*')
66+
@click.option('-r', '--ingest-report', is_flag=True,
67+
help='Create ingest report for updating other systems.')
6968
@click.pass_context
70-
def appendcoll(ctx, coll_handle, metadata_csv, file_path, file_type,
71-
ingest_type, ingest_report, multiple_terms):
69+
def additems(ctx, collection_handle, metadata_csv, field_map,
70+
directory, file_type, ingest_report):
7271
client = ctx.obj['client']
7372
start_time = ctx.obj['start_time']
74-
ingest_data = {}
75-
json_metadata = metadata.create_json_metadata(metadata_csv, multiple_terms)
76-
items = workflows.append_items_to_coll(client, coll_handle, ingest_type,
77-
file_path, file_type, json_metadata,
78-
ingest_data, ingest_report)
79-
for item in items:
80-
logger.info(f'Item posted: {item}')
73+
with open(metadata_csv, 'r') as csvfile, open(field_map, 'r') as jsonfile:
74+
metadata = csv.DictReader(csvfile)
75+
mapping = json.load(jsonfile)
76+
collection = Collection.from_csv(metadata, mapping)
77+
for item in collection.items:
78+
item.bitstreams_from_directory(directory, file_type)
79+
collection_uuid = client.get_id_from_handle(collection_handle)
80+
collection.handle = collection_handle
81+
collection.uuid = collection_uuid
82+
collection.post_items(client)
8183
helpers.elapsed_time(start_time, 'Total runtime:')
8284

83-
84-
@main.command()
85-
@click.option('-c', '--comm_handle', prompt='Enter the community handle',
86-
help='The handle of the community in which to create the ,'
87-
'collection.')
88-
@click.option('-n', '--coll_name', prompt='Enter the name of the collection',
89-
help='The name of the collection to be created.')
90-
@click.option('-m', '--metadata_csv', prompt='Enter the metadata CSV file',
91-
help='The path of the CSV file of metadata.')
92-
@click.option('-f', '--file_path', prompt='Enter the path',
93-
help='The path of the content, a URL or local drive path.')
94-
@click.option('-t', '--file_type', prompt='Enter the file type',
95-
help='The file type to be uploaded.')
96-
@click.option('-i', '--ingest_type', prompt='Enter the type of ingest',
97-
help='The type of ingest to perform: local, remote.',
98-
type=click.Choice(['local', 'remote']), default='remote')
99-
@click.option('-r', '--ingest_report', prompt='Create an ingest report?',
100-
help='Create ingest report for updating other systems',
101-
default=False)
102-
@click.option('-u', '--multiple_terms', prompt='Method of separating terms?',
103-
help='The way multiple terms are separated in the metadata CSV.',
104-
type=click.Choice(['delimited', 'num_columns']),
105-
default='delimited')
106-
@click.pass_context
107-
def newcoll(ctx, comm_handle, coll_name, metadata_csv, file_path, file_type,
108-
ingest_type, ingest_report, multiple_terms):
109-
client = ctx.obj['client']
110-
start_time = ctx.obj['start_time']
111-
ingest_data = {}
112-
json_metadata = metadata.create_json_metadata(metadata_csv, multiple_terms)
113-
items = workflows.populate_new_coll(client, comm_handle, coll_name,
114-
ingest_type, file_path, file_type,
115-
json_metadata, ingest_report,
116-
ingest_data)
117-
for item in items:
118-
logger.info(f'Item posted: {item}')
119-
if ingest_report == 'True':
120-
report_name = metadata_csv.replace('.csv', '-ingest.csv')
121-
helpers.create_ingest_report(ingest_data, report_name)
122-
helpers.elapsed_time(start_time, 'Total runtime:')
123-
124-
125-
@main.command()
126-
@click.option('-m', '--metadata_csv', prompt='Enter the metadata CSV file',
127-
help='The path of the CSV file of metadata.')
128-
@click.option('-o', '--output_path', prompt='Enter the output path',
129-
default='', help='The path of the output files, include '
130-
'/ at the end of the path')
131-
@click.option('-f', '--file_path', prompt='Enter the path',
132-
help='The path of the content, a URL or local drive path.'
133-
'Include / at the end of a local drive path.')
134-
@click.option('-t', '--file_type', prompt='Enter the file type',
135-
help='The file type to be uploaded.')
136-
def reconcile(metadata_csv, file_path, file_type, output_path):
137-
workflows.reconcile_files_and_metadata(metadata_csv, output_path,
138-
file_path, file_type)
85+
#
86+
# @main.command()
87+
# @click.option('-c', '--comm_handle', prompt='Enter the community handle',
88+
# help='The handle of the community in which to create the ,'
89+
# 'collection.')
90+
# @click.option('-n', '--coll_name', prompt='Enter the name of the collection',
91+
# help='The name of the collection to be created.')
92+
# @click.option('-m', '--metadata_csv', prompt='Enter the metadata CSV file',
93+
# help='The path of the CSV file of metadata.')
94+
# @click.option('-f', '--file_path', prompt='Enter the path',
95+
# help='The path of the content, a URL or local drive path.')
96+
# @click.option('-t', '--file_type', prompt='Enter the file type',
97+
# help='The file type to be uploaded.')
98+
# @click.option('-i', '--ingest_type', prompt='Enter the type of ingest',
99+
# help='The type of ingest to perform: local, remote.',
100+
# type=click.Choice(['local', 'remote']), default='remote')
101+
# @click.option('-r', '--ingest_report', prompt='Create an ingest report?',
102+
# help='Create ingest report for updating other systems',
103+
# default=False)
104+
# @click.option('-u', '--multiple_terms', prompt='Method of separating terms?',
105+
# help='The way multiple terms are separated in the metadata CSV.',
106+
# type=click.Choice(['delimited', 'num_columns']),
107+
# default='delimited')
108+
# @click.pass_context
109+
# def newcoll(ctx, comm_handle, coll_name, metadata_csv, file_path, file_type,
110+
# ingest_type, ingest_report, multiple_terms):
111+
# client = ctx.obj['client']
112+
# start_time = ctx.obj['start_time']
113+
# ingest_data = {}
114+
# json_metadata = metadata.create_json_metadata(metadata_csv, multiple_terms)
115+
# items = workflows.populate_new_coll(client, comm_handle, coll_name,
116+
# ingest_type, file_path, file_type,
117+
# json_metadata, ingest_report,
118+
# ingest_data)
119+
# for item in items:
120+
# logger.info(f'Item posted: {item}')
121+
# if ingest_report == 'True':
122+
# report_name = metadata_csv.replace('.csv', '-ingest.csv')
123+
# helpers.create_ingest_report(ingest_data, report_name)
124+
# helpers.elapsed_time(start_time, 'Total runtime:')
125+
#
126+
#
127+
# @main.command()
128+
# @click.option('-m', '--metadata_csv', prompt='Enter the metadata CSV file',
129+
# help='The path of the CSV file of metadata.')
130+
# @click.option('-o', '--output_path', prompt='Enter the output path',
131+
# default='', help='The path of the output files, include '
132+
# '/ at the end of the path')
133+
# @click.option('-f', '--file_path', prompt='Enter the path',
134+
# help='The path of the content, a URL or local drive path.'
135+
# 'Include / at the end of a local drive path.')
136+
# @click.option('-t', '--file_type', prompt='Enter the file type',
137+
# help='The file type to be uploaded.')
138+
# def reconcile(metadata_csv, file_path, file_type, output_path):
139+
# workflows.reconcile_files_and_metadata(metadata_csv, output_path,
140+
# file_path, file_type)
139141

140142

141143
if __name__ == '__main__':

‎dsaps/helpers.py‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,6 @@
1313
logger = structlog.get_logger()
1414

1515

16-
def build_file_dict_remote(directory_url, file_type, file_dict):
17-
"""Build list of files in a remote directory."""
18-
response = requests.get(directory_url)
19-
links = html.fromstring(response.content).iterlinks()
20-
for link in [i for i in links if i[2].endswith(file_type)]:
21-
file_identifier = link[2].replace(f'.{file_type}', '')
22-
file_dict[file_identifier] = f'{directory_url}{link[2]}'
23-
return file_dict
24-
2516

2617
def create_csv_from_list(list_name, output):
2718
"""Creates CSV file from list content."""

0 commit comments

Comments
 (0)