WIP feat(datasets): added dataset ls-files, unlink and delete commands #436
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
jirikuncar
reviewed
Mar 1, 2019
jirikuncar
reviewed
Mar 1, 2019
jirikuncar
reviewed
Mar 1, 2019
jirikuncar
reviewed
Mar 1, 2019
jirikuncar
reviewed
Mar 1, 2019
jirikuncar
reviewed
Mar 1, 2019
jirikuncar
reviewed
Mar 1, 2019
jirikuncar
reviewed
Mar 1, 2019
jirikuncar
reviewed
Mar 1, 2019
jirikuncar
reviewed
Mar 1, 2019
jirikuncar
reviewed
Mar 5, 2019
jirikuncar
reviewed
Mar 5, 2019
jirikuncar
reviewed
Mar 5, 2019
jirikuncar
reviewed
Mar 5, 2019
jirikuncar
reviewed
Mar 5, 2019
jirikuncar
reviewed
Mar 5, 2019
jirikuncar
reviewed
Mar 5, 2019
jirikuncar
reviewed
Mar 5, 2019
jirikuncar
reviewed
Mar 5, 2019
jirikuncar
reviewed
Mar 5, 2019
jirikuncar
reviewed
Mar 5, 2019
jirikuncar
reviewed
Mar 5, 2019
jirikuncar
reviewed
Mar 6, 2019
jirikuncar
reviewed
Mar 6, 2019
jirikuncar
reviewed
Mar 6, 2019
jirikuncar
reviewed
Mar 6, 2019
jirikuncar
reviewed
Mar 6, 2019
renku/api/datasets.py
Outdated
| files_in_dataset = [( | ||
| key, | ||
| os.path.realpath( | ||
| self.renku_datasets_path / dataset.name / dataset_file.path |
There was a problem hiding this comment.
Suggested change
| self.renku_datasets_path / dataset.name / dataset_file.path | |
| self.renku_datasets_path / dataset.identifier / dataset_file.path |
jirikuncar
reviewed
Mar 6, 2019
jirikuncar
reviewed
Mar 6, 2019
renku/api/datasets.py
Outdated
| meta_file = dataset_dir_path / self.METADATA | ||
| meta_file.unlink() | ||
|
|
||
| def unlink_files(self, dataset, pattern, exclude=None): |
There was a problem hiding this comment.
Suggested change
| def unlink_files(self, dataset, pattern, exclude=None): | |
| def unlink_files(self, dataset, include=None, exclude=None): |
jirikuncar
reviewed
Mar 6, 2019
renku/api/datasets.py
Outdated
|
|
||
| def _match(file): | ||
| filename = Path(file[1]).name | ||
| result = fnmatch(filename, pattern) |
There was a problem hiding this comment.
Suggested change
| result = fnmatch(filename, pattern) | |
| result = fnmatch(filename, include) if include is not None else result |
jirikuncar
reviewed
Mar 6, 2019
jirikuncar
reviewed
Mar 6, 2019
jirikuncar
reviewed
Mar 6, 2019
jirikuncar
reviewed
Mar 6, 2019
jirikuncar
reviewed
Mar 6, 2019
jirikuncar
reviewed
Mar 6, 2019
jirikuncar
reviewed
Mar 6, 2019
renku/cli/dataset.py
Outdated
| @click.option('--verbose', '-v', is_flag=True) | ||
| @pass_local_client(clean=True, commit=True) | ||
| def delete(client, name, force, verbose): | ||
| """Deletes a dataset.""" |
There was a problem hiding this comment.
Suggested change
| """Deletes a dataset.""" | |
| """Delete a dataset metadata.""" |
jirikuncar
reviewed
Mar 7, 2019
| @@ -46,11 +47,19 @@ class DatasetsApiMixin(object): | |||
| DATASETS = 'datasets' | |||
| """Directory for storing dataset metadata in Renku.""" | |||
|
|
|||
| REFS_DATASETS = 'refs' | |||
There was a problem hiding this comment.
why do you need it? the LinkReference object should provide you with enough abstraction.
jirikuncar
reviewed
Mar 7, 2019
| if not force and dataset.files: | ||
| raise errors.DatasetNotEmpty() | ||
|
|
||
| dataset_dir_path = self.renku_datasets_path / dataset.identifier.hex |
There was a problem hiding this comment.
you might need to iterate over all refs/datasets since there might be more aliases.
There was a problem hiding this comment.
we might need a another renku doctor check that find broken refs
jirikuncar
reviewed
Mar 7, 2019
| """Find datasets which contain given dataset file. | ||
|
|
||
| :param client: LocalClient instance. | ||
| :param dataset_file: DatasetFile instance. |
There was a problem hiding this comment.
IMHO it would be nicer to filter by path.
jirikuncar
reviewed
Mar 7, 2019
| @@ -54,3 +54,20 @@ def check_missing_files(client): | |||
| ) | |||
|
|
|||
| return False | |||
|
|
|||
|
|
|||
| def file_in_datasets(client, dataset_file, exclude_dataset=None): | |||
There was a problem hiding this comment.
consider building a map: path -> [dataset, ...]
jirikuncar
reviewed
Mar 7, 2019
| if exclude_dataset and \ | ||
| exclude_dataset.identifier == dataset.identifier: | ||
| continue | ||
| if dataset_file in dataset.files.values(): |
There was a problem hiding this comment.
Suggested change
| if dataset_file in dataset.files.values(): | |
| if dataset_file.path in dataset.files: |
jirikuncar
reviewed
Mar 7, 2019
| meta_file.unlink() | ||
| dataset_dir_path.rmdir() | ||
|
|
||
| dataset_ref = self.renku_refs_path / Path(dataset.name) |
There was a problem hiding this comment.
Suggested change
| dataset_ref = self.renku_refs_path / Path(dataset.name) |
jirikuncar
reviewed
Mar 7, 2019
| dataset_dir_path.rmdir() | ||
|
|
||
| dataset_ref = self.renku_refs_path / Path(dataset.name) | ||
| if dataset_ref.is_symlink(): |
There was a problem hiding this comment.
Suggested change
| if dataset_ref.is_symlink(): |
jirikuncar
reviewed
Mar 7, 2019
|
|
||
| dataset_ref = self.renku_refs_path / Path(dataset.name) | ||
| if dataset_ref.is_symlink(): | ||
| dataset_ref.unlink() |
There was a problem hiding this comment.
Suggested change
| dataset_ref.unlink() |
jirikuncar
reviewed
Mar 7, 2019
| relative_path, | ||
| os.path.realpath( | ||
| str( | ||
| self.renku_datasets_path / dataset.name / dataset_file.path |
There was a problem hiding this comment.
Suggested change
| self.renku_datasets_path / dataset.name / dataset_file.path | |
| self.renku_datasets_path / dataset.identifier.hex / dataset_file.path |
jirikuncar
reviewed
Mar 7, 2019
| return found_one | ||
|
|
||
| file_unlinked = [] | ||
| for relative, absolute, file_ in filter(_match, files_in_dataset): |
There was a problem hiding this comment.
using filter is not very "Pythonic". the better way is to use a list comprehension.
Suggested change
| for relative, absolute, file_ in filter(_match, files_in_dataset): | |
| file_unlinked = [ | |
| file_ | |
| for file_ in files_in_dataset | |
| if _match(file_) | |
| ] |
jirikuncar
reviewed
Mar 7, 2019
| :param include: Remove files matching the include pattern. | ||
| :param exclude: Keep files matching the exclude pattern. | ||
| """ | ||
| files_in_dataset = [( |
There was a problem hiding this comment.
files_in_dataset = [
self.renku_datasets_path / dataset.identifier.hex / file_
for file_ in dataset.files
]
jirikuncar
reviewed
Mar 7, 2019
| if not files_in_dataset: | ||
| raise errors.ResourceNotFound(resource_type='DatasetFile') | ||
|
|
||
| def _match(file): |
There was a problem hiding this comment.
Suggested change
| def _match(file): | |
| def _match(file_): |
jirikuncar
reviewed
Mar 7, 2019
| filename = Path(file[1]).name | ||
| found_one = False | ||
| for pattern in include: | ||
| found_one = fnmatch(filename, pattern) |
There was a problem hiding this comment.
Suggested change
| found_one = fnmatch(filename, pattern) | |
| found_one = file_.match(pattern) |
jsam
changed the title
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
DO NOT MERGE
This PR adds new commands for dataset:
Fixes #418