Merge branch 'master' into feature-1608-move-storage-to-core

.gitattributes

@@ -0,0 +1,3 @@
+*.po -diff
+*.mo -diff
+*.pot -diff

.tx/config

@@ -9,4 +9,5 @@ source_lang = en
 # Namings not quite the same in Transifex and babel:
 # Transifex vs Babel (& CKAN)
 # 'sr@Latin'  vs 'sr_Latn'
-trans.sr@Latin = translations/sr_Latn/LC_MESSAGES/ckan.po
+trans.sr@Latin = ckan/i18n/sr_Latn/LC_MESSAGES/ckan.po
+trans.sr@latin = ckan/i18n/sr_Latn/LC_MESSAGES/ckan.po

CHANGELOG.txt

@@ -1,8 +1,53 @@
 CKAN CHANGELOG
 ++++++++++++++
 
+v1.5.1 2011-01-04
+=================
+
+Major:
+ * Background tasks (#1363, #1371, #1408)
+ * Fix for security issue affecting CKAN v1.5 (#1585)
+
+Minor:
+ * Language support now excellent for European languages: en de fr it es no sv pl ru pt cs sr ca
+ * Web UI improvements:
+    * Resource editing refreshed
+    * Group editing refreshed
+    * Indication that group creation requires logging-in (#1004)
+    * Users' pictures displayed using Gravatar (#1409)
+    * 'Welcome' banner shown to new users (#1378)
+    * Group package list now ordered alphabetically (#1502)
+ * Allow managing a dataset's groups also via package entity API (#1381)
+ * Dataset listings in API standardised (#1490)
+ * Search ordering by modification and creation date (#191)
+ * Account creation disallowed with Open ID (create account in CKAN first) (#1386)
+ * User name can be modified (#1386)
+ * Email address required for registration (for password reset) (#1319)
+ * Atom feeds hidden for now
+ * New config options to ease CSS insertion into the template (#1380)
+ * Removed ETag browser cache headers (#1422)
+ * CKAN version number and admin contact in new 'status_show' API (#1087)
+ * Upgrade SQLAlchemy to 0.7.3 (#1433)
+ * SOLR schema is now versioned (#1498)
+
+Bug fixes:
+ * Group ordering on main page was alphabetical but should be by size (since 1.5) (#1487)
+ * Package could get added multiple times to same Group, distorting Group size (#1484)
+ * Search index corruption when multiple CKAN instances on a server all storing the same object (#1430)
+ * Dataset property metadata_created had wrong value (since v1.3) (#1546)
+ * Tag browsing showed tags for deleted datasets (#920)
+ * User name change field validation error (#1470)
+ * You couldn't edit a user with a unicode email address (#1479)
+ * Package search API results missed the extra fields (#1455)
+ * OpenID registration disablement explained better (#1532)
+ * Data upload (with ckanext-storage) failed if spaces in the filename (#1518)
+ * Resource download count fixed (integration with ckanext-googleanalytics) (#1451)
+
+
 v1.5 2011-11-07
 ===============
+**Deprecated due to security issue #1585**
+
 Major:
  * New visual theme (#1108)
     * Package & Resource edit overhaul (#1294/#1348/#1351/#1368/#1296)

bin/canada.py

@@ -0,0 +1,103 @@
+'''
+Script to sort out the tags imported from ca.ckan.net to thedatahub.org and
+got mangled in the process.
+'''
+
+import re
+from optparse import OptionParser
+import copy
+
+import ckanclient
+from status import Status
+
+def sort_out_tags(source_ckan_uri,
+                  dest_ckan_uri, dest_api_key,
+                  ):
+    ckan1 = ckanclient.CkanClient(base_location=source_ckan_uri)
+    ckan2 = ckanclient.CkanClient(base_location=dest_ckan_uri,
+                                  api_key=dest_api_key)
+
+    # ensure group exists
+    group = 'country-ca'
+    assert group in set(ckan2.group_register_get())
+    group_to_change = 'canadagov'
+
+    # work out tag mappings
+    tag_status = Status('tag mapping')
+    tag_replace_map = {}
+    source_tags = ckan1.tag_register_get()
+    for tag in source_tags:
+        mangled_tag = re.sub('[-._]', '', tag)
+        replacement_tag = tag
+        # Change underscores to hyphens
+        replacement_tag = replacement_tag.replace('_', '-')
+        # Remove trailing punctuation
+        if replacement_tag[-1] in '_-.':
+            replacement_tag = replacement_tag[:-1]
+        if replacement_tag[0] in '_-.':
+            replacement_tag = replacement_tag[1:]
+        if mangled_tag == replacement_tag:
+            tag_status.record('Unchanged', mangled_tag, do_print=False)
+            continue
+        if mangled_tag in tag_replace_map and tag_replace_map[mangled_tag] != replacement_tag:
+            print 'Warning - can\'t differentiate %s : %s / %s' % \
+                  (mangled_tag, tag_replace_map[mangled_tag], replacement_tag)
+        tag_status.record('Mapping added', '%s:%s' % (mangled_tag, replacement_tag), do_print=False)
+        tag_replace_map[mangled_tag] = replacement_tag
+    example_map = tag_replace_map.items()[0]
+    print tag_status
+
+    # Custom mappings
+    tag_replace_map['metaimportedfromcackannet'] = 'meta.imported-from-ca-ckan-net'
+
+    # edit packages
+    pkg_status = Status('Packages')
+    pkgs = ckan2.group_entity_get(group)['packages']
+    print 'Packages in the group: %i' % len(pkgs)
+    for pkg_name in pkgs:
+        pkg = ckan2.package_entity_get(pkg_name)
+        original_pkg = copy.deepcopy(pkg)
+
+        # Change tags
+        edited_tags = [tag_replace_map.get(tag, tag) for tag in pkg['tags']]
+        if 'canada' in edited_tags:
+            edited_tags.remove('canada')
+
+        if group_to_change in pkg['groups']:
+            pkg['groups'].remove(group_to_change)
+            edited_tags.append('canada-gov')
+
+        if set(pkg['tags']) != set(edited_tags):
+            pkg['tags'] = edited_tags
+            print '%s: %r -> %r' % (pkg_name, sorted(original_pkg['tags']), sorted(edited_tags))
+
+        if pkg == original_pkg:
+            pkg_status.record('Unchanged', pkg_name)
+            continue
+
+        try:
+            ckan2.package_entity_put(pkg)
+        except ckanclient.CkanApiError, e:
+            pkg_status.record('Error: %r' % e.args, pkg_name)
+            continue
+
+        pkg_status.record('Successfully changed', pkg_name)
+
+    print pkg_status
+
+usage = '''%prog [OPTIONS] <source_ckan_api_uri> <destination_ckan_api_uri>
+Recopy tags that got mangled in Canadian copy.'''
+parser = OptionParser(usage=usage)
+parser.add_option("-k", "--destination-ckan-api-key", dest="destination_ckan_api_key",
+                  help="Destination CKAN's API key", metavar="API-KEY")
+
+(options, args) = parser.parse_args()
+
+assert len(args) == 2, 'The source and destination CKAN API URIs are the only two arguments. Found: %r' % args
+source_ckan_uri, destination_ckan_uri = args
+print 'Key: ', options.destination_ckan_api_key
+
+sort_out_tags(source_ckan_uri,
+              destination_ckan_uri,
+              options.destination_ckan_api_key,
+)

bin/status.py

@@ -0,0 +1,26 @@
+from collections import defaultdict
+
+class Status:
+    '''When looping through objects and doing operations to them,
+    this is a useful object to keep track of what happens and
+    summarise the numbers at the end.'''
+    def __init__(self, obj_type_str=None):
+        self.obj_type_str = obj_type_str
+        self.pkg_status = defaultdict(list) # reason: [pkgs]
+
+    def record(self, status_category, pkg_name, do_print=True):
+        self.pkg_status[status_category].append(pkg_name)
+        if do_print:
+            print '%s: %s' % (pkg_name, status_category)
+
+    def __str__(self):
+        status = '\nStatus'
+        if self.obj_type_str:
+            status += ' of: %s' % self.obj_type_str
+        status += '\n'
+        status += '\n'.join([ \
+            '%s: %i (e.g. %s)' % (category, len(pkg_names), sorted(pkg_names)[0]) \
+            for (category, pkg_names) in self.pkg_status.items()])
+        status += '\nTotal: %i\n' % sum([len(pkg_names) for pkg_names in self.pkg_status.values()])
+        return status
+

build.sh

@@ -15,11 +15,25 @@
 # Over time, as CKAN and its dependencies follow conventions more closely the
 # edge cases handled by this script should go away and the build should become
 # simpler.
+#
+# Install:
+#
+#     sudo apt-get update
+#     sudo apt-get install -y wget
+#     echo "deb http://apt.3aims.com/buildkit-0.2.2 lucid universe" | sudo tee /etc/apt/sources.list.d/3aims.list
+#     wget -qO- "http://apt.3aims.com/packages_public.key" | sudo apt-key add -
+#     sudo apt-get update
+#     sudo apt-get install buildkit-deb buildkit-apt-repo
+
 
-CKAN_PACKAGE_VERSION=08
-DEPS_PACKAGE_VERSION=05
-CKAN_PATH=/home/james/Documents/Work/OKFN/code/pyenv/src/ckan
-BUILDKIT_REPO_PATH="/var/lib/buildkit/repo/ckan-1.5"
+CKAN_PACKAGE_VERSION=$1
+DEPS_PACKAGE_VERSION=$2
+# If you don't run this command from the CKAN source directory, specify the 
+# path to CKAN here
+CKAN_PATH=$PWD
+# You'll need to create the repo if it doesn't exist:
+# sudo -u buildkit buildkit repo clone base_lucid ckan-1.5.1
+REPO_NAME="ckan-1.5.1"
 PIP_DOWNLOAD_CACHE=${CKAN_PATH}/build/env/cache
 EMAIL=packaging@okfn.org
 NAME="James Gardner"
@@ -54,8 +68,7 @@ cp ${CKAN_PATH}/build/buildkit/env/build/licenses/dist/buildkit/*.deb ${CKAN_PAT
 echo "done."
 
 # Add the .debs to the repository and the export the latest files for upload
-echo "Adding the packages to the $BUILDKIT_REPO_PATH repo ..."
-sudo -u buildkit buildkit repo remove -a $BUILDKIT_REPO_PATH
-sudo -u buildkit buildkit repo add $BUILDKIT_REPO_PATH ${CKAN_PATH}/dist/buildkit/*.deb
-echo "done."
-
+# echo "Adding the packages to the $REPO_NAME repo using files in ${CKAN_PATH}/dist/buildkit/*.deb ..."
+# sudo -u buildkit buildkit repo remove -a $REPO_NAME dummy_arg
+# sudo -u buildkit buildkit repo add $REPO_NAME ${CKAN_PATH}/dist/buildkit/*.deb
+# echo "done."

ckan/__init__.py

@@ -1,17 +1,18 @@
 __version__ = '1.5.2a'
 __description__ = 'Comprehensive Knowledge Archive Network (CKAN) Software'
 __long_description__ = \
-'''The CKAN software is used to run the Comprehensive Knowledge Archive
-Network (CKAN) site: http://www.ckan.net.
+'''CKAN software provides a hub for datasets. The flagship site running CKAN 
+is theDataHub.org but it is also used for dozens of other open data websites
+run by governments, agencies and citizens.
 
-The Comprehensive Knowledge Archive Network is a registry of open
-knowledge packages and projects (and a few closed ones). CKAN is the
-place to search for open knowledge resources as well as register your
-own - be that a set of Shakespeare's works, a global population density
-database, the voting records of MPs, or 30 years of US patents.
+CKAN provides a place to search for open knowledge resources as well as 
+register your own - be that a set of Shakespeare's works, a global 
+population density database, the voting records of MPs, or 30 years of 
+US patents.
 
-Those familiar with freshmeat or CPAN can think of CKAN as providing an
-analogous service for open knowledge. 
+CKAN is an abbreviation for 'Comprehensive Knowledge Archive Network'. 
+Those familiar with Freshmeat or CPAN can think of CKAN as providing an
+analogous service for open data and knowledge. 
 '''
 __license__ = 'AGPL'
 

ckan/config/deployment.ini_tmpl

@@ -109,7 +109,7 @@ ckan.site_description =
 ckan.site_url =
 
 ## Favicon (default is the CKAN software favicon)
-ckan.favicon = http://assets.okfn.org/p/ckan/img/ckan.ico
+ckan.favicon = /images/icons/ckan.ico
 
 ## Solr support
 #solr_url = http://127.0.0.1:8983/solr

ckan/config/middleware.py

@@ -75,6 +75,14 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
 
     if asbool(config.get('openid_enabled', 'true')):
         from repoze.who.plugins.openid.identification import OpenIdIdentificationPlugin
+        # Monkey patches for repoze.who.openid
+        # Fixes #1659 - enable log-out when CKAN mounted at non-root URL 
+        from ckan.lib import repoze_patch
+        OpenIdIdentificationPlugin.identify = repoze_patch.identify
+        OpenIdIdentificationPlugin.redirect_to_logged_in = repoze_patch.redirect_to_logged_in
+        OpenIdIdentificationPlugin._redirect_to_loginform = repoze_patch._redirect_to_loginform
+        OpenIdIdentificationPlugin.challenge = repoze_patch.challenge
+
         who_parser.identifiers = [i for i in who_parser.identifiers if \
                 not isinstance(i, OpenIdIdentificationPlugin)]
         who_parser.challengers = [i for i in who_parser.challengers if \

ckan/controllers/group.py

@@ -14,6 +14,7 @@
 from ckan.logic import check_access, get_action
 from ckan.logic.schema import group_form_schema
 from ckan.logic import tuplize_dict, clean_dict, parse_params
+from ckan.lib.dictization.model_dictize import package_dictize
 import ckan.forms
 
 class GroupController(BaseController):
@@ -95,15 +96,21 @@ def read(self, id):
         c.group_description_formatted = desc_formatted
         c.group_admins = self.authorizer.get_admins(c.group)
 
+        context['return_query'] = True
         results = get_action('group_package_show')(context, data_dict)
 
         c.page = Page(
             collection=results,
             page=request.params.get('page', 1),
             url=h.pager_url,
-            items_per_page=50
+            items_per_page=30
         )
 
+        result = []
+        for pkg_rev in c.page.items:
+            result.append(package_dictize(pkg_rev, context))
+        c.page.items = result
+
         return render('group/read.html')
 
     def new(self, data=None, errors=None, error_summary=None):

ckan/controllers/package.py

@@ -141,11 +141,15 @@ def pager_url(q=None, page=None):
 
         try:
             c.fields = []
+            search_extras = {}
             for (param, value) in request.params.items():
                 if not param in ['q', 'page'] \
                         and len(value) and not param.startswith('_'):
-                    c.fields.append((param, value))
-                    q += ' %s: "%s"' % (param, value)
+                    if not param.startswith('ext_'):
+                        c.fields.append((param, value))
+                        q += ' %s: "%s"' % (param, value)
+                    else:
+                        search_extras[param] = value
 
             context = {'model': model, 'session': model.Session,
                        'user': c.user or c.author}
@@ -155,6 +159,7 @@ def pager_url(q=None, page=None):
                 'facet.field':g.facets,
                 'rows':limit,
                 'start':(page-1)*limit,
+                'extras':search_extras
             }
 
             query = get_action('package_search')(context,data_dict)

ckan/controllers/user.py

@@ -240,6 +240,12 @@ def _save_edit(self, id, context):
     def login(self):
         if 'error' in request.params:
             h.flash_error(request.params['error'])
+
+        if request.environ['SCRIPT_NAME'] and g.openid_enabled:
+            # #1662 restriction
+            log.warn('Cannot mount CKAN at a URL and login with OpenID.')
+            g.openid_enabled = False
+
         return render('user/login.html')
 
     def logged_in(self):
@@ -260,7 +266,8 @@ def logged_in(self):
             h.flash_success(_("%s is now logged in") % user_dict['display_name'])
             return self.me()
         else:
-            h.flash_error(_('Login failed. Bad username or password.'))
+            h.flash_error('Login failed. Bad username or password.' + \
+                          ' (Or if using OpenID, it hasn\'t been associated with a user account.)')
             h.redirect_to(controller='user', action='login')
 
     def logged_out(self):
@@ -307,7 +314,7 @@ def request_reset(self):
                 try:
                     mailer.send_reset_link(user_obj)
                     h.flash_success(_('Please check your inbox for a reset code.'))
-                    redirect('/')
+                    h.redirect_to('/')
                 except mailer.MailerException, e:
                     h.flash_error(_('Could not send reset link: %s') % unicode(e))
         return render('user/request_reset.html')
@@ -338,7 +345,7 @@ def perform_reset(self, id):
                 user = get_action('user_update')(context, user_dict)
 
                 h.flash_success(_("Your password has been reset."))
-                redirect('/')
+                h.redirect_to('/')
             except NotAuthorized:
                 h.flash_error(_('Unauthorized to edit user %s') % id)
             except NotFound, e: