feat(datasets): added ls-files command

Author: jsam

renku/cli/_format/dataset_files.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2019 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Serializers for dataset list files."""
+
+from collections import OrderedDict
+
+from renku.cli._echo import echo_via_pager
+
+
+def tabular(client, records):
+    """Format dataset files with a tabular output.
+
+    :param client: LocalClient instance.
+    :param records: Filtered collection.
+    """
+    from renku.models._tabulate import tabulate
+
+    echo_via_pager(
+        tabulate(
+            records,
+            headers=OrderedDict((
+                ('added', None),
+                ('authors_csv', 'authors'),
+                ('dataset', None),
+                ('full_path', 'path'),
+            )),
+        )
+    )
+
+
+def jsonld(client, records):
+    """Format dataset files as JSON-LD.
+
+    :param client: LocalClient instance.
+    :param records: Filtered collection.
+    """
+    from renku.models._json import dumps
+    from renku.models._jsonld import asjsonld
+
+    data = [asjsonld(record) for record in records]
+    echo_via_pager(dumps(data, indent=2))
+
+
+FORMATS = {
+    'tabular': tabular,
+    'json-ld': jsonld,
+}
+"""Valid formatting options."""

renku/cli/dataset.py

@@ -83,6 +83,7 @@
 
 from ._client import pass_local_client
 from ._echo import progressbar
+from ._format.dataset_files import FORMATS as DATASET_FILES_FORMATS
 from ._format.datasets import FORMATS as DATASETS_FORMATS
 
 
@@ -157,6 +158,107 @@ def add(client, name, urls, link, relative_to, target, force):
         raise BadParameter('Could not process {0}'.format(url))
 
 
+@dataset.command('ls-files')
+@click.option(
+    '--dataset', multiple=True, help='Filter files in specific dataset.'
+)
+@click.option(
+    '--authors',
+    help='Filter files which where authored by specific authors. '
+    'Multiple authors are specified by comma.'
+)
+@click.option(
+    '-I',
+    '--include',
+    default=None,
+    multiple=True,
+    help='Include files matching given pattern.'
+)
+@click.option(
+    '-X',
+    '--exclude',
+    default=None,
+    multiple=True,
+    help='Exclude files matching given pattern.'
+)
+@click.option(
+    '--format',
+    type=click.Choice(DATASET_FILES_FORMATS),
+    default='tabular',
+    help='Choose an output format.'
+)
+@pass_local_client(clean=False, commit=False)
+def ls_files(client, format, exclude, include, authors, dataset):
+    """List files in dataset."""
+    records = _filter(
+        client,
+        dataset_names=dataset,
+        authors=authors,
+        include=include,
+        exclude=exclude
+    )
+
+    DATASET_FILES_FORMATS[format](client, records)
+
+
+def _include_exclude(file_path, include=None, exclude=None):
+    """Check if file matches one of include filters and not in exclude filter.
+
+    :param file_path: Path to the file.
+    :param include: Tuple containing patterns to which include from result.
+    :param exclude: Tuple containing patterns to which exclude from result.
+    """
+    if exclude is not None and exclude:
+        for pattern in exclude:
+            if file_path.match(pattern):
+                return False
+
+    if include is not None and include:
+        for pattern in include:
+            if file_path.match(pattern):
+                return True
+        return False
+
+    return True
+
+
+def _filter(
+    client, dataset_names=None, authors=None, include=None, exclude=None
+):
+    """Filter dataset files by specified filters.
+
+    :param dataset_names: Filter by specified dataset names.
+    :param authors: Filter by authors.
+    :param include: Include files matching file pattern.
+    :param exclude: Exclude files matching file pattern.
+    """
+    if isinstance(authors, str):
+        authors = set(authors.split(','))
+
+    if isinstance(authors, list) or isinstance(authors, tuple):
+        authors = set(authors)
+
+    records = []
+    for path_, dataset in client.datasets.items():
+        if dataset.name in dataset_names or not dataset_names:
+            for file_ in dataset.files.values():
+                file_.dataset = dataset.name
+
+                path_ = file_.full_path.relative_to(client.path)
+                match = _include_exclude(path_, include, exclude)
+
+                if authors:
+                    match = match and authors.issubset({
+                        author.name
+                        for author in file_.authors
+                    })
+
+                if match:
+                    records.append(file_)
+
+    return sorted(records, key=lambda file_: file_.added)
+
+
 def get_datadir():
     """Fetch the current data directory."""
     ctx = click.get_current_context()

renku/models/datasets.py

@@ -19,6 +19,7 @@
 
 import configparser
 import datetime
+import os
 import re
 import uuid
 from functools import partial
@@ -100,27 +101,45 @@ def from_commit(cls, commit):
         )
 
 
+@attr.s
+class AuthorsMixin:
+    """Mixin for handling authors container."""
+
+    authors = jsonld.container.list(Author, kw_only=True)
+
+    @property
+    def authors_csv(self):
+        """Comma-separated list of authors associated with dataset."""
+        return ",".join(author.name for author in self.authors)
+
+
 @jsonld.s(
     type='http://schema.org/DigitalDocument',
     slots=True,
 )
-class DatasetFile(object):
+class DatasetFile(AuthorsMixin):
     """Represent a file in a dataset."""
 
-    path = _path_attr()
+    path = _path_attr(kw_only=True)
     url = jsonld.ib(
-        default=None,
-        context='http://schema.org/url',
+        default=None, context='http://schema.org/url', kw_only=True
     )
-    authors = jsonld.container.list(Author)
-    dataset = attr.ib(default=None)
-    added = jsonld.ib(context='http://schema.org/dateCreated', )
+    authors = jsonld.container.list(Author, kw_only=True)
+    dataset = attr.ib(default=None, kw_only=True)
+    added = jsonld.ib(context='http://schema.org/dateCreated', kw_only=True)
 
     @added.default
     def _now(self):
         """Define default value for datetime fields."""
         return datetime.datetime.utcnow()
 
+    @property
+    def full_path(self):
+        """Return full path in the current reference frame."""
+        return Path(
+            os.path.realpath(str(self.__reference__.parent / self.path))
+        )
+
 
 def _parse_date(value):
     """Convert date to datetime."""
@@ -148,7 +167,7 @@ def _convert_dataset_files(value):
         'scoro': 'http://purl.org/spar/scoro/',
     },
 )
-class Dataset(object):
+class Dataset(AuthorsMixin):
     """Repesent a dataset."""
 
     SUPPORTED_SCHEMES = ('', 'file', 'http', 'https', 'git+https', 'git+ssh')