Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 30 additions & 32 deletions renku/cli/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -316,10 +316,8 @@ def dataset(ctx, revision, datadir, format):


@dataset.command()
@click.argument('name')
@click.option(
'--short-name', default='', help='A convenient name for dataset.'
)
@click.argument('short_name')
@click.option('--title', default='', help='Title of the dataset.')
@click.option(
'-d', '--description', default='', help='Dataset\'s description.'
)
Expand All @@ -330,13 +328,13 @@ def dataset(ctx, revision, datadir, format):
multiple=True,
help='Creator\'s name and email ("Name <email>").'
)
def create(name, short_name, description, creator):
def create(short_name, title, description, creator):
"""Create an empty dataset in the current repo."""
creators = creator or ()

new_dataset = create_dataset(
name=name,
short_name=short_name,
title=title,
description=description,
creators=creators
)
Expand All @@ -361,7 +359,7 @@ def edit(dataset_id):


@dataset.command()
@click.argument('name')
@click.argument('short_name')
@click.argument('urls', nargs=-1)
@click.option('--link', is_flag=True, help='Creates a hard link.')
@click.option(
Expand Down Expand Up @@ -393,12 +391,12 @@ def edit(dataset_id):
@click.option(
'--ref', default=None, help='Add files from a specific commit/tag/branch.'
)
def add(name, urls, link, force, create, sources, destination, ref):
def add(short_name, urls, link, force, create, sources, destination, ref):
"""Add data to a dataset."""
progress = partial(progressbar, label='Adding data to dataset')
add_file(
urls=urls,
name=name,
short_name=short_name,
link=link,
force=force,
create=create,
Expand All @@ -410,7 +408,7 @@ def add(name, urls, link, force, create, sources, destination, ref):


@dataset.command('ls-files')
@click.argument('names', nargs=-1)
@click.argument('short_names', nargs=-1)
@click.option(
'--creators',
help='Filter files which where authored by specific creators. '
Expand All @@ -436,13 +434,13 @@ def add(name, urls, link, force, create, sources, destination, ref):
default='tabular',
help='Choose an output format.'
)
def ls_files(names, creators, include, exclude, format):
def ls_files(short_names, creators, include, exclude, format):
"""List files in dataset."""
echo_via_pager(list_files(names, creators, include, exclude, format))
echo_via_pager(list_files(short_names, creators, include, exclude, format))


@dataset.command()
@click.argument('name')
@click.argument('short_name')
@click.option(
'-I',
'--include',
Expand All @@ -458,13 +456,13 @@ def ls_files(names, creators, include, exclude, format):
@click.option(
'-y', '--yes', is_flag=True, help='Confirm unlinking of all files.'
)
def unlink(name, include, exclude, yes):
def unlink(short_name, include, exclude, yes):
"""Remove matching files from a dataset."""
with file_unlink(name, include, exclude) as records:
with file_unlink(short_name, include, exclude) as records:
if not yes and records:
prompt_text = (
'You are about to remove '
'following from "{0}" dataset.\n'.format(dataset.name) +
'following from "{0}" dataset.\n'.format(short_name) +
'\n'.join([str(record.full_path) for record in records]) +
'\nDo you wish to continue?'
)
Expand All @@ -474,8 +472,8 @@ def unlink(name, include, exclude, yes):


@dataset.command('rm')
@click.argument('names', nargs=-1)
def remove(names):
@click.argument('short_names', nargs=-1)
def remove(short_names):
"""Delete a dataset."""
datasetscontext = partial(
progressbar,
Expand All @@ -488,7 +486,7 @@ def remove(names):
item_show_func=lambda item: item.name if item else '',
)
dataset_remove(
names,
short_names,
with_output=True,
datasetscontext=datasetscontext,
referencescontext=referencescontext
Expand All @@ -497,38 +495,38 @@ def remove(names):


@dataset.command('tag')
@click.argument('name')
@click.argument('short_name')
@click.argument('tag')
@click.option(
'-d', '--description', default='', help='A description for this tag'
)
@click.option('--force', is_flag=True, help='Allow overwriting existing tags.')
def tag(name, tag, description, force):
def tag(short_name, tag, description, force):
"""Create a tag for a dataset."""
tag_dataset_with_client(name, tag, description, force)
tag_dataset_with_client(short_name, tag, description, force)
click.secho('OK', fg='green')


@dataset.command('rm-tags')
@click.argument('name')
@click.argument('short_name')
@click.argument('tags', nargs=-1)
def remove_tags(name, tags):
def remove_tags(short_name, tags):
"""Remove tags from a dataset."""
remove_dataset_tags(name, tags)
remove_dataset_tags(short_name, tags)
click.secho('OK', fg='green')


@dataset.command('ls-tags')
@click.argument('name')
@click.argument('short_name')
@click.option(
'--format',
type=click.Choice(DATASET_TAGS_FORMATS),
default='tabular',
help='Choose an output format.'
)
def ls_tags(name, format):
def ls_tags(short_name, format):
"""List all tags of a dataset."""
tags_output = list_tags(name, format)
tags_output = list_tags(short_name, format)
click.echo(tags_output)


Expand Down Expand Up @@ -565,7 +563,7 @@ def export_(id, provider, publish, tag):
@dataset.command('import')
@click.argument('uri')
@click.option(
'--short-name', default='', help='A convenient name for dataset.'
'--short-name', default=None, help='A convenient name for dataset.'
)
@click.option(
'-x',
Expand Down Expand Up @@ -612,7 +610,7 @@ def finalize(self):


@dataset.command('update')
@click.argument('names', nargs=-1)
@click.argument('short_names', nargs=-1)
@click.option(
'--creators',
help='Filter files which where authored by specific creators. '
Expand Down Expand Up @@ -640,11 +638,11 @@ def finalize(self):
is_flag=True,
help='Delete local files that are deleted from remote.'
)
def update(names, creators, include, exclude, ref, delete):
def update(short_names, creators, include, exclude, ref, delete):
"""Updates files in dataset from a remote Git repo."""
progress_context = partial(progressbar, label='Updating files')
update_datasets(
names=names,
short_names=short_names,
creators=creators,
include=include,
exclude=exclude,
Expand Down
62 changes: 32 additions & 30 deletions renku/core/commands/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,11 @@ def dataset_parent(client, revision, datadir, format, ctx=None):
)
def create_dataset(
client,
name,
short_name=None,
description=None,
short_name,
title=None,
description='',
creators=None,
commit_message=None,
commit_message=None
):
"""Create an empty dataset in the current repo.

Expand All @@ -89,8 +89,8 @@ def create_dataset(
creators = [Person.from_dict(creator) for creator in creators]

dataset, _, __ = client.create_dataset(
name=name,
short_name=short_name,
title=title,
description=description,
creators=creators
)
Expand Down Expand Up @@ -124,7 +124,7 @@ def edit_dataset(client, dataset_id, transform_fn, commit_message=None):
def add_file(
client,
urls,
name,
short_name,
link=False,
force=False,
create=False,
Expand All @@ -139,7 +139,7 @@ def add_file(
add_to_dataset(
client=client,
urls=urls,
short_name=name,
short_name=short_name,
link=link,
force=force,
create=create,
Expand Down Expand Up @@ -219,11 +219,11 @@ def add_to_dataset(


@pass_local_client(clean=False, commit=False)
def list_files(client, names, creators, include, exclude, format):
def list_files(client, short_names, creators, include, exclude, format):
"""List files in dataset."""
records = _filter(
client,
names=names,
short_names=short_names,
creators=creators,
include=include,
exclude=exclude
Expand All @@ -238,15 +238,15 @@ def list_files(client, names, creators, include, exclude, format):
commit_only=COMMIT_DIFF_STRATEGY,
)
@contextmanager
def file_unlink(client, name, include, exclude, commit_message=None):
def file_unlink(client, short_name, include, exclude, commit_message=None):
"""Remove matching files from a dataset."""
dataset = client.load_dataset(name=name)
dataset = client.load_dataset(short_name=short_name)

if not dataset:
raise ParameterError('Dataset does not exist.')

records = _filter(
client, names=[dataset.name], include=include, exclude=exclude
client, short_names=[short_name], include=include, exclude=exclude
)
if not records:
raise ParameterError('No records found.')
Expand All @@ -266,18 +266,18 @@ def file_unlink(client, name, include, exclude, commit_message=None):
)
def dataset_remove(
client,
names,
short_names,
with_output=False,
datasetscontext=contextlib.nullcontext,
referencescontext=contextlib.nullcontext,
commit_message=None
):
"""Delete a dataset."""
datasets = {name: client.get_dataset_path(name) for name in names}
datasets = {name: client.get_dataset_path(name) for name in short_names}

if not datasets:
raise ParameterError(
'use dataset name or identifier', param_hint='names'
'use dataset short_name or identifier', param_hint='short_names'
)

unknown = [
Expand All @@ -286,7 +286,7 @@ def dataset_remove(
]
if unknown:
raise ParameterError(
'unknown datasets ' + ', '.join(unknown), param_hint='names'
'unknown datasets ' + ', '.join(unknown), param_hint='short_names'
)

datasets = set(datasets.values())
Expand Down Expand Up @@ -496,7 +496,7 @@ def import_dataset(
)
def update_datasets(
client,
names,
short_names,
creators,
include,
exclude,
Expand All @@ -508,7 +508,7 @@ def update_datasets(
"""Update files from a remote Git repo."""
records = _filter(
client,
names=names,
short_names=short_names,
creators=creators,
include=include,
exclude=exclude
Expand All @@ -527,7 +527,7 @@ def update_datasets(
dataset = datasets.get(dataset_name)

if not dataset:
dataset = client.load_dataset(name=dataset_name)
dataset = client.load_dataset(short_name=dataset_name)
datasets[dataset_name] = dataset

file_.dataset = dataset
Expand Down Expand Up @@ -576,10 +576,12 @@ def _include_exclude(file_path, include=None, exclude=None):
return True


def _filter(client, names=None, creators=None, include=None, exclude=None):
def _filter(
client, short_names=None, creators=None, include=None, exclude=None
):
"""Filter dataset files by specified filters.

:param names: Filter by specified dataset names.
:param short_names: Filter by specified dataset short_names.
:param creators: Filter by creators.
:param include: Include files matching file pattern.
:param exclude: Exclude files matching file pattern.
Expand All @@ -592,7 +594,7 @@ def _filter(client, names=None, creators=None, include=None, exclude=None):

records = []
for path_, dataset in client.datasets.items():
if not names or dataset.short_name in names:
if not short_names or dataset.short_name in short_names:
for file_ in dataset.files:
file_.dataset = dataset.name
path_ = file_.full_path.relative_to(client.path)
Expand All @@ -616,15 +618,15 @@ def _filter(client, names=None, creators=None, include=None, exclude=None):
commit_only=COMMIT_DIFF_STRATEGY,
)
def tag_dataset_with_client(
client, name, tag, description, force=False, commit_message=None
client, short_name, tag, description, force=False, commit_message=None
):
"""Creates a new tag for a dataset and injects a LocalClient."""
tag_dataset(client, name, tag, description, force)
tag_dataset(client, short_name, tag, description, force)


def tag_dataset(client, name, tag, description, force=False):
def tag_dataset(client, short_name, tag, description, force=False):
"""Creates a new tag for a dataset."""
dataset_ = client.load_dataset(name)
dataset_ = client.load_dataset(short_name)
if not dataset_:
raise ParameterError('Dataset not found.')

Expand All @@ -641,9 +643,9 @@ def tag_dataset(client, name, tag, description, force=False):
commit=True,
commit_only=COMMIT_DIFF_STRATEGY,
)
def remove_dataset_tags(client, name, tags, commit_message=True):
def remove_dataset_tags(client, short_name, tags, commit_message=True):
"""Removes tags from a dataset."""
dataset = client.load_dataset(name)
dataset = client.load_dataset(short_name)
if not dataset:
raise ParameterError('Dataset not found.')

Expand All @@ -656,9 +658,9 @@ def remove_dataset_tags(client, name, tags, commit_message=True):


@pass_local_client(clean=False, commit=False)
def list_tags(client, name, format):
def list_tags(client, short_name, format):
"""List all tags for a dataset."""
dataset_ = client.load_dataset(name)
dataset_ = client.load_dataset(short_name)

if not dataset_:
raise ParameterError('Dataset not found.')
Expand Down
Loading