Merge branch 'master' of github.com:okfn/ckan into 3016-template-tweaks

ckan/i18n/check_po_files.py

@@ -11,7 +11,6 @@
 
 '''
 import re
-import polib
 import paste.script.command
 
 def simple_conv_specs(s):
@@ -98,6 +97,8 @@ class CheckPoFiles(paste.script.command.Command):
     parser = paste.script.command.Command.standard_parser(verbose=True)
 
     def command(self):
+        import polib
+
         test_simple_conv_specs()
         test_mapping_keys()
         test_replacement_fields()

ckan/lib/app_globals.py

@@ -29,6 +29,38 @@
     'ckan.site_custom_css',
 ]
 
+config_details = {
+    'ckan.favicon': {}, # default gets set in config.environment.py
+    'ckan.template_head_end': {},
+    'ckan.template_footer_end': {},
+        # has been setup in load_environment():
+    'ckan.site_id': {},
+    'ckan.recaptcha.publickey': {'name': 'recaptcha_publickey'},
+    'ckan.recaptcha.privatekey': {'name': 'recaptcha_publickey'},
+    'ckan.template_title_deliminater': {'default': '-'},
+    'ckan.template_head_end': {},
+    'ckan.template_footer_end': {},
+    'ckan.dumps_url': {},
+    'ckan.dumps_format': {},
+    'ckan.api_url': {},
+
+    # split string
+    'search.facets': {'default': 'groups tags res_format license',
+                      'type': 'split',
+                      'name': 'facets'},
+    'package_hide_extras': {'type': 'split'},
+    'plugins': {'type': 'split'},
+
+    # bool
+    'openid_enabled': {'default': 'true', 'type' : 'bool'},
+    'debug': {'default': 'false', 'type' : 'bool'},
+    'ckan.debug_supress_header' : {'default': 'false', 'type' : 'bool'},
+
+    # int
+    'ckan.datasets_per_page': {'default': '20', 'type': 'int'},
+}
+
+
 # A place to store the origional config options of we override them
 _CONFIG_CACHE = {}
 
@@ -139,29 +171,27 @@ def _check_uptodate(self):
                 self._mutex.release()
 
     def _init(self):
-        self.favicon = config.get('ckan.favicon', '/images/icons/ckan.ico')
-        facets = config.get('search.facets', 'groups tags res_format license')
-        self.facets = facets.split()
 
-        # has been setup in load_environment():
-        self.site_id = config.get('ckan.site_id')
-
-        self.template_head_end = config.get('ckan.template_head_end', '')
-        self.template_footer_end = config.get('ckan.template_footer_end', '')
-
-        # hide these extras fields on package read
-        package_hide_extras = config.get('package_hide_extras', '').split()
-        self.package_hide_extras = package_hide_extras
-
-        self.openid_enabled = asbool(config.get('openid_enabled', 'true'))
+        # process the config_details to set globals
+        for name, options in config_details.items():
+            if 'name' in options:
+                key = options['name']
+            elif name.startswith('ckan.'):
+                key = name[5:]
+            else:
+                key = name
+            value = config.get(name, options.get('default', ''))
 
-        self.recaptcha_publickey = config.get('ckan.recaptcha.publickey', '')
-        self.recaptcha_privatekey = config.get('ckan.recaptcha.privatekey', '')
+            data_type = options.get('type')
+            if data_type == 'bool':
+                value = asbool(value)
+            elif data_type == 'int':
+                value = int(value)
+            elif data_type == 'split':
+                value = value.split()
 
-        datasets_per_page = int(config.get('ckan.datasets_per_page', '20'))
-        self.datasets_per_page = datasets_per_page
+            setattr(self, key, value)
 
-        self.debug_supress_header = asbool(config.get('ckan.debug_supress_header', 'false'))
 
 app_globals = _Globals()
 del _Globals

ckan/lib/base.py

@@ -128,6 +128,10 @@ def render_template():
 
         # Jinja2 templates
         if template_type == 'jinja2':
+            # We don't want to have the config in templates it should be
+            # accessed via g (app_globals) as this gives us flexability such
+            # as changing via database settings.
+            del globs['config']
             # TODO should we raise error if genshi filters??
             return render_jinja2(template_name, globs)
 

ckan/lib/cli.py

@@ -1,3 +1,5 @@
+import collections
+import csv
 import os
 import datetime
 import sys
@@ -578,9 +580,12 @@ class UserCmd(CkanCommand):
       user                            - lists users
       user list                       - lists users
       user <user-name>                - shows user properties
-      user add <user-name> [apikey=<apikey>] [password=<password>]
-                                      - add a user (prompts for password if
-                                        not supplied)
+      user add <user-name> [<field>=<value>]
+                                      - add a user (prompts for password
+                                        if not supplied).
+                                        Field can be: apikey
+                                                      password
+                                                      email
       user setpass <user-name>        - set user password (prompts)
       user remove <user-name>         - removes user from users
       user search <query>             - searches for a user name
@@ -677,53 +682,37 @@ def add(self):
 
         if len(self.args) < 2:
             print 'Need name of the user.'
-            return
-        username = self.args[1]
-        user = model.User.by_name(unicode(username))
-        if user:
-            print 'User "%s" already found' % username
             sys.exit(1)
+        username = self.args[1]
 
-        # parse args
-        apikey = None
-        password = None
-        args = self.args[2:]
-        if len(args) == 1 and not (args[0].startswith('password') or \
-                                   args[0].startswith('apikey')):
-            # continue to support the old syntax of just supplying
-            # the apikey
-            apikey = args[0]
-        else:
-            # new syntax: password=foo apikey=bar
-            for arg in args:
-                split = arg.find('=')
-                if split == -1:
-                    split = arg.find(' ')
-                    if split == -1:
-                        raise ValueError('Could not parse arg: %r (expected "--<option>=<value>)")' % arg)
-                key, value = arg[:split], arg[split+1:]
-                if key == 'password':
-                    password = value
-                elif key == 'apikey':
-                    apikey = value
-                else:
-                    raise ValueError('Could not parse arg: %r (expected password/apikey argument)' % arg)
+        # parse args into data_dict
+        data_dict = {'name': username}
+        for arg in self.args[2:]:
+            try:
+                field, value = arg.split('=', 1)
+                data_dict[field] = value
+            except ValueError:
+                raise ValueError('Could not parse arg: %r (expected "<option>=<value>)"' % arg)
 
-        if not password:
-            password = self.password_prompt()
+        if 'password' not in data_dict:
+            data_dict['password'] = self.password_prompt()
 
         print('Creating user: %r' % username)
 
-
-        user_params = {'name': unicode(username),
-                       'password': password}
-        if apikey:
-            user_params['apikey'] = unicode(apikey)
-        user = model.User(**user_params)
-        model.Session.add(user)
-        model.repo.commit_and_remove()
-        user = model.User.by_name(unicode(username))
-        print user
+        try:
+            import ckan.logic as logic
+            site_user = logic.get_action('get_site_user')({'model': model, 'ignore_auth': True}, {})
+            context = {
+                    'model': model,
+                    'session': model.Session,
+                    'ignore_auth': True,
+                    'user': site_user['name'],
+                    }
+            user_dict = logic.get_action('user_create')(context, data_dict)
+            pprint(user_dict)
+        except logic.ValidationError, e:
+            print e
+            sys.exit(1)
 
     def remove(self):
         import ckan.model as model
@@ -942,30 +931,52 @@ def clean(self, user_ratings=True):
             rating.purge()
         model.repo.commit_and_remove()
 
+
+## Used by the Tracking class
+_ViewCount = collections.namedtuple("ViewCount", "id name count")
+
+
 class Tracking(CkanCommand):
     '''Update tracking statistics
 
     Usage:
-      tracking   - update tracking stats
+      tracking update [start-date]        - update tracking stats
+      tracking export <file> [start-date] - export tracking stats to a csv file
     '''
 
     summary = __doc__.split('\n')[0]
     usage = __doc__
-    max_args = 1
-    min_args = 0
+    max_args = 3
+    min_args = 1
 
     def command(self):
         self._load_config()
         import ckan.model as model
         engine = model.meta.engine
 
-        if len(self.args) == 1:
-            # Get summeries from specified date
-            start_date = datetime.datetime.strptime(self.args[0], '%Y-%m-%d')
+        cmd = self.args[0]
+        if cmd == 'update':
+            start_date = self.args[1] if len(self.args) > 1 else None
+            self.update_all(engine, start_date)
+        elif cmd == 'export':
+            if len(self.args) <= 1:
+                print self.__class__.__doc__
+                sys.exit(1)
+            output_file = self.args[1]
+            start_date = self.args[2] if len(self.args) > 2 else None
+            self.update_all(engine, start_date)
+            self.export_tracking(engine, output_file)
+        else:
+            print self.__class__.__doc__
+            sys.exit(1)
+
+    def update_all(self, engine, start_date=None):
+        if start_date:
+            start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d')
         else:
             # No date given. See when we last have data for and get data
             # from 2 days before then in case new data is available.
-            # If no date here then use 2010-01-01 as the start date
+            # If no date here then use 2011-01-01 as the start date
             sql = '''SELECT tracking_date from tracking_summary
                      ORDER BY tracking_date DESC LIMIT 1;'''
             result = engine.execute(sql).fetchall()
@@ -985,6 +996,56 @@ def command(self):
             print 'tracking updated for %s' % start_date
             start_date = stop_date
 
+    def _total_views(self, engine):
+        sql = '''
+            SELECT p.id,
+                   p.name,
+                   COALESCE(SUM(s.count), 0) AS total_views
+               FROM package AS p
+               LEFT OUTER JOIN tracking_summary AS s ON s.package_id = p.id
+               GROUP BY p.id, p.name
+               ORDER BY total_views DESC
+        '''
+        return [_ViewCount(*t) for t in engine.execute(sql).fetchall()]
+
+    def _recent_views(self, engine, measure_from):
+        sql = '''
+            SELECT p.id,
+                   p.name,
+                   COALESCE(SUM(s.count), 0) AS total_views
+               FROM package AS p
+               LEFT OUTER JOIN tracking_summary AS s ON s.package_id = p.id
+               WHERE s.tracking_date >= %(measure_from)s
+               GROUP BY p.id, p.name
+               ORDER BY total_views DESC
+        '''
+        return [_ViewCount(*t) for t in engine.execute(
+                    sql, measure_from=str(measure_from)
+                ).fetchall()]
+
+    def export_tracking(self, engine, output_filename):
+        '''Write tracking summary to a csv file.'''
+        HEADINGS = [
+            "dataset id",
+            "dataset name",
+            "total views",
+            "recent views (last 2 weeks)",
+        ]
+
+        measure_from = datetime.date.today() - datetime.timedelta(days=14)
+        recent_views = self._recent_views(engine, measure_from)
+        total_views = self._total_views(engine)
+
+        with open(output_filename, 'w') as fh:
+            f_out = csv.writer(fh)
+            f_out.writerow(HEADINGS)
+            recent_views_for_id = dict((r.id, r.count) for r in recent_views)
+            f_out.writerows([(r.id,
+                              r.name,
+                              r.count,
+                              recent_views_for_id.get(r.id, 0))
+                              for r in total_views])
+
     def update_tracking(self, engine, summary_date):
         PACKAGE_URL = '/dataset/'
         # clear out existing data before adding new

ckan/templates/base.html

@@ -10,6 +10,10 @@
 <!--[if gt IE 8]><!--> <html lang="{{ lang }}"> <!--<![endif]-->
 {%- endblock -%}
 
+<!-- 
+{{ g }}
+-->
+
   {# Allows custom attributes to be added to the <head> tag #}
   <head{% block headtag %}{% endblock %}>
     {#
@@ -41,7 +45,7 @@
     <title>
       {%- block title -%}
         {%- block subtitle %}{% endblock -%}
-        {%- if self.subtitle()|trim %} {{ config.template_title_delimiter or '-' }} {% endif -%}
+        {%- if self.subtitle()|trim %} {{ g.template_title_deliminater }} {% endif -%}
         {{ g.site_title }}
       {%- endblock -%}
     </title>
@@ -51,9 +55,7 @@
     such as rss feeds and favicons in the same way as the meta block.
     #}
     {% block links -%}
-    {%- with favicon = config.get('ckan.favicon') or h.url_for_static('/base/images/ckan.ico') -%}
-      <link rel="shortcut icon" href="{{ favicon }}" />
-    {% endwith -%}
+      <link rel="shortcut icon" href="{{ g.favicon }}" />
     {% endblock -%}
 
     {#
@@ -72,9 +74,9 @@
       {% resource g.main_css[6:] %}
     {% endblock %}
 
-    {# defined in the config.ini under "ckan.template_head_end" #}
     {% block head_extras %}
-      {{ config.get('ckan.template_head_end', '') | safe }}
+      {# defined in the config.ini under "ckan.template_head_end" #}
+      {{ g.template_head_end | safe }}
     {% endblock %}
 
     {%- block custom_styles %}
@@ -87,7 +89,7 @@
   </head>
 
   {# Allows custom attributes to be added to the <body> tag #}
-  <body{% block bodytag %} data-site-root="{{ h.url('/', locale='default', qualified=true) }}" data-locale-root="{{ h.url('/', qualified=true) }}" data-api-root="{{ config.get('ckan.api_url', '') }}"{% endblock %}>
+  <body{% block bodytag %} data-site-root="{{ h.url('/', locale='default', qualified=true) }}" data-locale-root="{{ h.url('/', qualified=true) }}" data-api-root="{{ g.api_url }}"{% endblock %}>
 
     {#
     The page block allows you to add content to the page. Most of the time it is
@@ -119,9 +121,9 @@
     {%- block scripts %}
     {% endblock -%}
 
-    {# defined in the config.ini under "ckan.template_footer_end" #}
     {% block body_extras -%}
-      {{ config.get('ckan.template_footer_end', '') | safe }}
+      {# defined in the config.ini under "ckan.template_footer_end" #}
+      {{ g.template_footer_end | safe }}
     {%- endblock %}
   </body>
 </html>