Merge branch '2204-resource-views-cli' into test-cli-autodoc

ckan/lib/cli.py

@@ -6,6 +6,9 @@
 import sys
 from pprint import pprint
 import re
+import itertools
+import json
+import logging
 import ckan.logic as logic
 import ckan.model as model
 import ckan.include.rjsmin as rjsmin
@@ -15,6 +18,7 @@
 import sqlalchemy as sa
 import urlparse
 import routes
+from pylons import config
 
 import paste.script
 from paste.registry import Registry
@@ -2152,38 +2156,332 @@ def command(self):
         cmd.args = (root, ckanext)
         cmd.command()
 
+
 class ViewsCommand(CkanCommand):
     '''Manage resource views.
 
     Usage:
 
-        paster views create all                 - Create views for all types.
-        paster views create [type1] [type2] ... - Create views for specified types.
-        paster views clean                      - Permanently delete views for all types no longer in the configuration file.
+        paster views create [options] [type1] [type2] ...
+
+            Create views on relevant resources. You can optionally provide
+            specific view types (eg `recline_view`, `image_view`). If no types
+            are provided, the default ones will be used. These are generally
+            the ones defined in the `ckan.views.default_views` config option.
+            Note that on either case, plugins must be loaded (ie added to
+            `ckan.plugins`), otherwise the command will stop.
+
+        paster views clean
+
+            Permanently delete views for all types no longer present in the
+            `ckan.plugins` configuration option.
 
-    Supported types are "pdf", "text", "webpage", "image" and "grid".  Make
-    sure the relevant plugins are loaded for the following types, otherwise
-    an error will be raised:
-        * "grid"-> "recline_grid_view"
-        * "pdf" -> "pdf_view"
-        * "text -> "text_view"
     '''
 
     summary = __doc__.split('\n')[0]
     usage = __doc__
     min_args = 1
 
+    def __init__(self, name):
+
+        super(ViewsCommand, self).__init__(name)
+
+        self.parser.add_option('-y', '--yes', dest='assume_yes',
+                               action='store_true',
+                               default=False,
+                               help='''Automatic yes to prompts. Assume "yes"
+as answer to all prompts and run non-interactively''')
+
+        self.parser.add_option('-d', '--dataset', dest='dataset_id',
+                               action='append',
+                               help='''Create views on a particular dataset.
+You can use the dataset id or name, and it can be defined multiple times.''')
+
+        self.parser.add_option('--no-default-filters',
+                               dest='no_default_filters',
+                               action='store_true',
+                               default=False,
+                               help='''Do not add default filters for relevant
+resource formats for the view types provided. Note that filters are not added
+by default anyway if an unsupported view type is provided or when using the
+`-s` or `-d` options.''')
+
+        self.parser.add_option('-s', '--search', dest='search_params',
+                               action='store',
+                               default=False,
+                               help='''Extra search parameters that will be
+used for getting the datasets to create the resource views on. It must be a
+JSON object like the one used by the `package_search` API call. Supported
+fields are `q`, `fq` and `fq_list`. Check the documentation for examples.
+Not used when using the `-d` option.''')
+
     def command(self):
         self._load_config()
         if not self.args:
             print self.usage
         elif self.args[0] == 'create':
-            self.create_views(self.args[1:])
+            view_plugin_types = self.args[1:]
+            self.create_views_search(view_plugin_types)
+            # self.create_views(self.args[1:])
         elif self.args[0] == 'clean':
             self.clean_views()
         else:
             print self.usage
 
+    _page_size = 100
+
+    def _get_view_plugins(self, view_plugin_types,
+                          get_datastore_views=False):
+        '''
+        Returns the view plugins that were succesfully loaded
+
+        Views are provided as a list of ``view_plugin_types``. If no types are
+        provided, the default views defined in the ``ckan.views.default_views``
+        will be created. Only in this case (when the default view plugins are
+        used) the `get_datastore_views` parameter can be used to get also view
+        plugins that require data to be in the DataStore.
+
+        If any of the provided plugins could not be loaded (eg it was not added
+        to `ckan.plugins`) the command will stop.
+
+        Returns a list of loaded plugin names.
+        '''
+        from ckan.lib.datapreview import (get_view_plugins,
+                                          get_default_view_plugins
+                                          )
+
+        log = logging.getLogger(__name__)
+
+        view_plugins = []
+
+        if not view_plugin_types:
+            log.info('No view types provided, using default types')
+            view_plugins = get_default_view_plugins()
+            if get_datastore_views:
+                view_plugins.extend(
+                    get_default_view_plugins(get_datastore_views=True))
+        else:
+            view_plugins = get_view_plugins(view_plugin_types)
+
+        loaded_view_plugins = [view_plugin.info()['name']
+                               for view_plugin in view_plugins]
+
+        plugins_not_found = list(set(view_plugin_types) -
+                                 set(loaded_view_plugins))
+
+        if plugins_not_found:
+            msg = ('View plugin(s) not found : {0}. '.format(plugins_not_found)
+                   + 'Have they been added to the `ckan.plugins` configuration'
+                   + ' option?')
+            log.error(msg)
+            sys.exit(1)
+
+        return loaded_view_plugins
+
+    def _add_default_filters(self, search_data_dict, view_types):
+        '''
+        Adds extra filters to the `package_search` dict for common view types
+
+        It basically adds `fq` parameters that filter relevant resource formats
+        for the view types provided. For instance, if one of the view types is
+        `pdf_view` the following will be added to the final query:
+
+            fq=res_format:"pdf" OR res_format:"PDF"
+
+        This obviously should only be used if all view types are known and can
+        be filtered, otherwise we want all datasets to be returned. If a
+        non-filterable view type is provided, the search params are not
+        modified.
+
+        Returns the provided data_dict for `package_search`, optionally
+        modified with extra filters.
+        '''
+
+        from ckanext.imageview.plugin import DEFAULT_IMAGE_FORMATS
+        from ckanext.textview.plugin import get_formats as get_text_formats
+        from ckanext.datapusher.plugin import DEFAULT_FORMATS as \
+            datapusher_formats
+
+        filter_formats = []
+
+        for view_type in view_types:
+            if view_type == 'image_view':
+
+                for _format in DEFAULT_IMAGE_FORMATS:
+                    filter_formats.extend([_format, _format.upper()])
+
+            elif view_type == 'text_view':
+                formats = get_text_formats(config)
+                for _format in itertools.chain.from_iterable(formats.values()):
+                    filter_formats.extend([_format, _format.upper()])
+
+            elif view_type == 'pdf_view':
+                filter_formats.extend(['pdf', 'PDF'])
+
+            elif view_type in ['recline_view', 'recline_grid_view',
+                               'recline_graph_view', 'recline_map_view']:
+
+                if datapusher_formats[0] in filter_formats:
+                    continue
+
+                for _format in datapusher_formats:
+                    if '/' not in _format:
+                        filter_formats.extend([_format, _format.upper()])
+            else:
+                # There is another view type provided so we can't add any
+                # filter
+                return search_data_dict
+
+        filter_formats_query = ['+res_format:"{0}"'.format(_format)
+                                for _format in filter_formats]
+        search_data_dict['fq_list'].append(' OR '.join(filter_formats_query))
+
+        return search_data_dict
+
+    def _update_search_params(self, search_data_dict):
+        '''
+        Update the `package_search` data dict with the user provided parameters
+
+        Supported fields are `q`, `fq` and `fq_list`.
+
+        If the provided JSON object can not be parsed the process stops with
+        an error.
+
+        Returns the updated data dict
+        '''
+
+        log = logging.getLogger(__name__)
+
+        if not self.options.search_params:
+            return search_data_dict
+
+        try:
+            user_search_params = json.loads(self.options.search_params)
+        except ValueError, e:
+            log.error('Unable to parse JSON search parameters: {0}'.format(e))
+            sys.exit(1)
+
+        if user_search_params.get('q'):
+            search_data_dict['q'] = user_search_params['q']
+
+        if user_search_params.get('fq'):
+            if search_data_dict['fq']:
+                search_data_dict['fq'] += ' ' + user_search_params['fq']
+            else:
+                search_data_dict['fq'] = user_search_params['fq']
+
+        if (user_search_params.get('fq_list') and
+                isinstance(user_search_params['fq_list'], list)):
+            search_data_dict['fq_list'].extend(user_search_params['fq_list'])
+
+    def _search_datasets(self, page=1, view_types=[]):
+        '''
+        Perform a query with `package_search` and return the result
+
+        Results can be paginated using the `page` parameter
+        '''
+
+        n = self._page_size
+
+        search_data_dict = {
+            'q': '',
+            'fq': '',
+            'fq_list': [],
+            'rows': n,
+            'start': n * (page - 1),
+        }
+
+        if (not self.options.no_default_filters and
+                not self.options.search_params and
+                not self.options.dataset_id):
+            self._add_default_filters(search_data_dict, view_types)
+
+        if (self.options.search_params and
+                not self.options.dataset_id):
+            self._update_search_params(search_data_dict)
+
+        if self.options.dataset_id:
+            search_data_dict['q'] = ' OR '.join(
+                ['id:{0} OR name:"{0}"'.format(dataset_id)
+                 for dataset_id in self.options.dataset_id]
+            )
+
+        if not search_data_dict.get('q'):
+            search_data_dict['q'] = '*:*'
+
+        if ('dataset_type:dataset' not in search_data_dict['fq'] and
+                'dataset_type:dataset' not in search_data_dict['fq_list']):
+            search_data_dict['fq_list'].append('dataset_type:dataset')
+
+        query = p.toolkit.get_action('package_search')(
+            {'ignore_capacity_check': True},
+            search_data_dict)
+
+        return query
+
+    def create_views_search(self, view_plugin_types=[]):
+
+        from ckan.lib.datapreview import add_views_to_dataset_resources
+
+        log = logging.getLogger(__name__)
+
+        datastore_enabled = 'datastore' in config['ckan.plugins'].split()
+
+        loaded_view_plugins = self._get_view_plugins(view_plugin_types,
+                                                     datastore_enabled)
+
+        context = {'user': self.site_user['name']}
+
+        page = 1
+        while True:
+            query = self._search_datasets(page, loaded_view_plugins)
+
+            if page == 1 and query['count'] == 0:
+                log.info('No datasets to create resource views on, exiting...')
+                sys.exit(1)
+
+            elif page == 1 and not self.options.assume_yes:
+
+                msg = ('\nYou are about to check {0} datasets for the ' +
+                       'following view plugins: {1}\n' +
+                       ' Do you want to continue?')
+
+                confirm = query_yes_no(msg.format(query['count'],
+                                                  loaded_view_plugins))
+
+                if confirm == 'no':
+                    log.info('Command aborted by user')
+                    sys.exit(1)
+
+            if query['results']:
+                for dataset_dict in query['results']:
+
+                    if not dataset_dict.get('resources'):
+                        continue
+
+                    views = add_views_to_dataset_resources(
+                        context,
+                        dataset_dict,
+                        view_types=loaded_view_plugins)
+
+                    if views:
+                        view_types = list(set([view['view_type']
+                                               for view in views]))
+                        msg = ('Added {0} view(s) of type(s) {1} to ' +
+                               'resources from dataset {2}')
+                        log.debug(msg.format(len(views),
+                                             ', '.join(view_types),
+                                             dataset_dict['name']))
+
+                if len(query['results']) < self._page_size:
+                    break
+
+                page += 1
+            else:
+                break
+
+        log.info('Done')
+
     def create_views(self, view_types):
         supported_types = ['grid', 'text', 'webpage', 'pdf', 'image']
         if not view_types: