Merge branch 'release-v1.7.1-ecportal' of github.com:okfn/ckan into r…

…elease-v1.7.1-ecportal

ckan/controllers/group.py

@@ -316,6 +316,7 @@ def _save_edit(self, id, context):
             context['message'] = data_dict.get('log_message', '')
             data_dict['id'] = id
             group = get_action('group_update')(context, data_dict)
+
             h.redirect_to('%s_read' % str(group['type']), id=group['name'])
         except NotAuthorized:
             abort(401, _('Unauthorized to read group %s') % id)

ckan/lib/cli.py

@@ -1,3 +1,5 @@
+import collections
+import csv
 import os
 import datetime
 import sys
@@ -428,14 +430,14 @@ def export_datasets(self, out_folder):
             url = h.url_for( controller='package',action='read',
                                                   id=dd['name'])
 
-            url = urlparse.urljoin(fetch_url, url[1:]) + '.rdf'
+            url = urlparse.urljoin(fetch_url, url) + '.rdf'
             try:
                 fname = os.path.join( out_folder, dd['name'] ) + ".rdf"
                 r = urllib2.urlopen(url).read()
                 with open(fname, 'wb') as f:
                     f.write(r)
             except IOError, ioe:
-                sys.stderr.write( str(ioe) + "\n" )
+                sys.stderr.write(url + ":\t" + str(ioe) + "\n" )
 
 
 
@@ -884,30 +886,50 @@ def clean(self, user_ratings=True):
             rating.purge()
         model.repo.commit_and_remove()
 
+## Used by the Tracking class
+_ViewCount = collections.namedtuple("ViewCount", "id name count")
+
 class Tracking(CkanCommand):
     '''Update tracking statistics
 
     Usage:
-      tracking   - update tracking stats
+      tracking update [start-date]          - update tracking stats
+      tracking export <file> [start-date]   - export tracking stats to a csv file
     '''
 
     summary = __doc__.split('\n')[0]
     usage = __doc__
-    max_args = 1
-    min_args = 0
+    max_args = 3
+    min_args = 1
 
     def command(self):
         self._load_config()
         import ckan.model as model
         engine = model.meta.engine
 
-        if len(self.args) == 1:
-            # Get summeries from specified date
-            start_date = datetime.datetime.strptime(self.args[0], '%Y-%m-%d')
+        cmd = self.args[0]
+        if cmd == 'update':
+            start_date = self.args[1] if len(self.args) > 1 else None
+            self.update_all(engine, start_date)
+        elif cmd == 'export':
+            if len(self.args) <= 1:
+                print self.__class__.__doc__
+                exit(1)
+            output_file = self.args[1]
+            start_date = self.args[2] if len(self.args) > 2 else None
+            self.update_all(engine, start_date)
+            self.export_tracking(engine, output_file)
+        else:
+            print self.__class__.__doc__
+            exit(1)
+
+    def update_all(self, engine, start_date=None):
+        if start_date:
+            start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d')
         else:
             # No date given. See when we last have data for and get data
             # from 2 days before then in case new data is available.
-            # If no date here then use 2010-01-01 as the start date
+            # If no date here then use 2011-01-01 as the start date
             sql = '''SELECT tracking_date from tracking_summary
                      ORDER BY tracking_date DESC LIMIT 1;'''
             result = engine.execute(sql).fetchall()
@@ -927,6 +949,61 @@ def command(self):
             print 'tracking updated for %s' % start_date
             start_date = stop_date
 
+    def _total_views(self, engine):
+        sql = '''
+            SELECT p.id,
+                   p.name,
+                   COALESCE(SUM(s.count), 0) AS total_views
+               FROM package AS p
+               LEFT OUTER JOIN tracking_summary AS s ON s.package_id = p.id
+               GROUP BY p.id, p.name
+               ORDER BY total_views DESC
+        '''
+
+        return [ _ViewCount(*t) for t in engine.execute(sql).fetchall() ]
+
+    def _recent_views(self, engine, measure_from):
+
+        sql = '''
+            SELECT p.id,
+                   p.name,
+                   COALESCE(SUM(s.count), 0) AS total_views
+               FROM package AS p
+               LEFT OUTER JOIN tracking_summary AS s ON s.package_id = p.id
+               WHERE s.tracking_date >= %(measure_from)s
+               GROUP BY p.id, p.name
+               ORDER BY total_views DESC
+        '''
+
+        return [ _ViewCount(*t) for t in engine.execute(
+                        sql,
+                        measure_from=str(measure_from)
+                    ).fetchall() ]
+
+    def export_tracking(self, engine, output_filename):
+        '''Write tracking summary to a csv file.'''
+
+        HEADINGS = [
+                "dataset id",
+                "dataset name",
+                "total views",
+                "recent views (last 2 weeks)",
+        ]
+
+        measure_from = datetime.date.today() - datetime.timedelta(days=14)
+        recent_views = self._recent_views(engine, measure_from)
+        total_views = self._total_views(engine)
+
+        with open(output_filename, 'w') as fh:
+            f_out = csv.writer(fh)
+            f_out.writerow(HEADINGS)
+            recent_views_for_id = dict( (r.id, r.count) for r in recent_views )
+            f_out.writerows([ (r.id,
+                               r.name,
+                               r.count,
+                               recent_views_for_id.get(r.id, 0))
+                                    for r in total_views ])
+
     def update_tracking(self, engine, summary_date):
         PACKAGE_URL = '/dataset/'
         # clear out existing data before adding new
@@ -963,33 +1040,33 @@ def update_tracking(self, engine, summary_date):
         # update summary totals for resources
         sql = '''UPDATE tracking_summary t1
                  SET running_total = (
-                    SELECT sum(count)
+                    SELECT COALESCE(sum(count), 0)
                     FROM tracking_summary t2
                     WHERE t1.url = t2.url
-                    AND t2.tracking_date <= t1.tracking_date
+                    AND t2.tracking_date < t1.tracking_date
                  ) + t1.count
                  ,recent_views = (
-                    SELECT sum(count)
+                    SELECT COALESCE(sum(count), 0)
                     FROM tracking_summary t2
                     WHERE t1.url = t2.url
-                    AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - 14
+                    AND t2.tracking_date < t1.tracking_date AND t2.tracking_date >= t1.tracking_date - 14
                  ) + t1.count
                  WHERE t1.running_total = 0 AND tracking_type = 'resource';'''
         engine.execute(sql)
 
         # update summary totals for pages
         sql = '''UPDATE tracking_summary t1
                  SET running_total = (
-                    SELECT sum(count)
+                    SELECT COALESCE(sum(count), 0)
                     FROM tracking_summary t2
                     WHERE t1.package_id = t2.package_id
-                    AND t2.tracking_date <= t1.tracking_date
+                    AND t2.tracking_date < t1.tracking_date
                  ) + t1.count
                  ,recent_views = (
-                    SELECT sum(count)
+                    SELECT COALESCE(sum(count), 0)
                     FROM tracking_summary t2
                     WHERE t1.package_id = t2.package_id
-                    AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - 14
+                    AND t2.tracking_date < t1.tracking_date AND t2.tracking_date >= t1.tracking_date - 14
                  ) + t1.count
                  WHERE t1.running_total = 0 AND tracking_type = 'page'
                  AND t1.package_id IS NOT NULL

ckan/lib/helpers.py

@@ -10,6 +10,7 @@
 import logging
 import re
 import urllib
+import urlparse
 
 from paste.deploy.converters import asbool
 from webhelpers.html import escape, HTML, literal, url_escape
@@ -65,6 +66,7 @@ def url(*args, **kw):
 def url_for(*args, **kw):
     """Create url adding i18n information if selected
     wrapper for routes.url_for"""
+
     locale = kw.pop('locale', None)
     # remove __ckan_no_root and add after to not pollute url
     no_root = kw.pop('__ckan_no_root', False)
@@ -76,6 +78,16 @@ def url_for(*args, **kw):
         # fix ver to include the slash
         kw['ver'] = '/%s' % ver
     my_url = _routes_default_url_for(*args, **kw)
+
+    # _add_i18n_to_url() only works on the assumption that the `url_to_amend`
+    # startswith the mount-point of the application.  This will already be
+    # taken care of by `routes` if we're within a http request.  Otherwise, we
+    # have to make a best guess.
+    if not _handling_request():
+        # Safe to assume `kw['qualified'] is False` since `routes` would have
+        # thrown an exception by now if it were `True`.
+        my_url = _guess_mount_point() + my_url
+
     kw['__ckan_no_root'] = no_root
     return _add_i18n_to_url(my_url, locale=locale, **kw)
 
@@ -152,6 +164,31 @@ def _add_i18n_to_url(url_to_amend, **kw):
 
     return url
 
+def _guess_mount_point():
+    '''
+    Returns the best guess of the application's mount-point assuming we are
+    running *outside* of an http request.
+
+    The mount-point of the application can only be reliably found from the
+    SCRIPT_NAME wsgi environment variable.  But if we're outside of an http
+    request, then this variable is not available.  So we make a best guess
+    using the `site_url` config option.
+    '''
+    site_url = config.get('ckan.site_url')
+    return urlparse.urlparse(site_url).path.rstrip('/')
+
+def _handling_request():
+    '''Returns `True` iff we can detect that we're handling a http request.
+
+    Url-generation is not contained to callers handling http requests, it can
+    be called upon outside of a http request, for example, a paster command.
+    '''
+    try:
+        request.environ.get('SCRIPT_NAME', '')
+        return True
+    except TypeError:
+        return False
+
 def lang():
     ''' Reurn the language code for the current locale eg `en` '''
     return request.environ.get('CKAN_LANG')

ckan/lib/search/__init__.py

@@ -105,6 +105,19 @@ class SynchronousSearchPlugin(SingletonPlugin):
     implements(IDomainObjectModification, inherit=True)
 
     def notify(self, entity, operation):
+
+        if (isinstance(entity, model.Group) and
+                operation == DomainObjectOperation.changed):
+            if 'name' in entity.diff():
+                for dataset in entity.active_packages().all():
+                    dispatch_by_operation(
+                            dataset.__class__.__name__,
+                            get_action('package_show')(
+                                {'model': model, 'ignore_auth': True, 'validate': False},
+                                {'id': dataset.id}),
+                            operation
+                    )
+
         if not isinstance(entity, model.Package):
             return
         if operation != DomainObjectOperation.deleted:

ckan/logic/action/update.py

@@ -526,6 +526,7 @@ def term_translation_update_many(context, data_dict):
 
     context['defer_commit'] = True
 
+    num = 0
     for num, row in enumerate(data_dict['data']):
         term_translation_update(context, row)
 

ckan/logic/auth/publisher/delete.py

@@ -20,6 +20,10 @@ def package_delete(context, data_dict):
     package = get_package_object(context, data_dict)
     userobj = model.User.get( user )
 
+    if Authorizer().is_sysadmin(unicode(user)):
+        return {'success': True}
+
+
     if not userobj or \
        not _groups_intersect( userobj.get_groups('organization'), package.get_groups('organization') ):
         return {'success': False,
@@ -63,6 +67,9 @@ def group_delete(context, data_dict):
     if not user:
         return {'success': False, 'msg': _('Only members of this group are authorized to delete this group')}
 
+    if Authorizer().is_sysadmin(unicode(user)):
+        return {'success': True}
+
     group = get_group_object(context, data_dict)
     userobj = model.User.get( user )
     if not userobj:

ckan/model/modification.py

@@ -11,6 +11,7 @@
 from ckan.model.extension import ObserverNotifier
 from ckan.model.domain_object import DomainObjectOperation
 
+from ckan.model.group import Group
 from ckan.model.package import Package
 from ckan.model.resource import ResourceGroup, Resource
 from ckan.model.package_extra import PackageExtra
@@ -41,13 +42,13 @@ def before_commit(self, session):
         deleted = obj_cache['deleted']
 
         for obj in set(new):
-            if isinstance(obj, (Package, Resource)):
+            if isinstance(obj, (Package, Resource, Group)):
                 self.notify(obj, DomainObjectOperation.new)
         for obj in set(deleted):
-            if isinstance(obj, (Package, Resource)):
+            if isinstance(obj, (Package, Resource, Group)):
                 self.notify(obj, DomainObjectOperation.deleted)
         for obj in set(changed):
-            if isinstance(obj, Resource):
+            if isinstance(obj, (Resource, Group)):
                 self.notify(obj, DomainObjectOperation.changed)
             if getattr(obj, 'url_changed', False):
                 for item in PluginImplementations(IResourceUrlChange):

ckanext/multilingual/plugin.py

@@ -128,6 +128,8 @@ def before_index(self, search_data):
                 all_terms.extend(value)
             elif value in (None, True, False, {}):
                 continue
+            elif not isinstance(value, basestring):
+                value = unicode(value)
             else:
                 all_terms.append(value)