Merge branch 'master' into feature-1485-package-and-group-custom-forms

Conflicts:
	ckan/lib/dictization/model_dictize.py
	ckan/logic/schema.py
	ckan/tests/functional/test_cors.py

.gitattributes

@@ -0,0 +1,3 @@
+*.po -diff
+*.mo -diff
+*.pot -diff

.tx/config

@@ -9,4 +9,5 @@ source_lang = en
 # Namings not quite the same in Transifex and babel:
 # Transifex vs Babel (& CKAN)
 # 'sr@Latin'  vs 'sr_Latn'
-trans.sr@Latin = translations/sr_Latn/LC_MESSAGES/ckan.po
+trans.sr@Latin = ckan/i18n/sr_Latn/LC_MESSAGES/ckan.po
+trans.sr@latin = ckan/i18n/sr_Latn/LC_MESSAGES/ckan.po

CHANGELOG.txt

@@ -1,8 +1,53 @@
 CKAN CHANGELOG
 ++++++++++++++
 
+v1.5.1 2011-01-04
+=================
+
+Major:
+ * Background tasks (#1363, #1371, #1408)
+ * Fix for security issue affecting CKAN v1.5 (#1585)
+
+Minor:
+ * Language support now excellent for European languages: en de fr it es no sv pl ru pt cs sr ca
+ * Web UI improvements:
+    * Resource editing refreshed
+    * Group editing refreshed
+    * Indication that group creation requires logging-in (#1004)
+    * Users' pictures displayed using Gravatar (#1409)
+    * 'Welcome' banner shown to new users (#1378)
+    * Group package list now ordered alphabetically (#1502)
+ * Allow managing a dataset's groups also via package entity API (#1381)
+ * Dataset listings in API standardised (#1490)
+ * Search ordering by modification and creation date (#191)
+ * Account creation disallowed with Open ID (create account in CKAN first) (#1386)
+ * User name can be modified (#1386)
+ * Email address required for registration (for password reset) (#1319)
+ * Atom feeds hidden for now
+ * New config options to ease CSS insertion into the template (#1380)
+ * Removed ETag browser cache headers (#1422)
+ * CKAN version number and admin contact in new 'status_show' API (#1087)
+ * Upgrade SQLAlchemy to 0.7.3 (#1433)
+ * SOLR schema is now versioned (#1498)
+
+Bug fixes:
+ * Group ordering on main page was alphabetical but should be by size (since 1.5) (#1487)
+ * Package could get added multiple times to same Group, distorting Group size (#1484)
+ * Search index corruption when multiple CKAN instances on a server all storing the same object (#1430)
+ * Dataset property metadata_created had wrong value (since v1.3) (#1546)
+ * Tag browsing showed tags for deleted datasets (#920)
+ * User name change field validation error (#1470)
+ * You couldn't edit a user with a unicode email address (#1479)
+ * Package search API results missed the extra fields (#1455)
+ * OpenID registration disablement explained better (#1532)
+ * Data upload (with ckanext-storage) failed if spaces in the filename (#1518)
+ * Resource download count fixed (integration with ckanext-googleanalytics) (#1451)
+
+
 v1.5 2011-11-07
 ===============
+**Deprecated due to security issue #1585**
+
 Major:
  * New visual theme (#1108)
     * Package & Resource edit overhaul (#1294/#1348/#1351/#1368/#1296)

MANIFEST.in

@@ -5,6 +5,9 @@ recursive-include ckan/config *.xml
 recursive-include ckan/i18n *
 recursive-include ckan/templates *
 recursive-include ckan *.ini
+recursive-include ckanext/**/public *
+recursive-include ckanext/**/templates *
+recursive-include ckanext/**/i18n *
 prune .git
 include CHANGELOG.txt
 include ckan/migration/migrate.cfg

README.rst

@@ -0,0 +1,22 @@
+CKAN is open-source data hub software. CKAN makes it easy to publish, share and
+work with data. It's a data management system that provides a powerful platform
+for cataloging, storing and accessing datasets with a rich front-end, full API
+(for both data and catalog), visualization tools and more. Read more at
+http://ckan.org/. 
+
+ * Installation instructions: see docs at http://docs.ckan.org/
+ * Project wiki: http://wiki.ckan.org/
+ * Developer mailing list: ckan-dev@lists.okfn.org
+ * Issue tracker: http://trac.ckan.org/
+
+
+Copying and License
+===================
+
+This material is copyright (c) 2006-2011 Open Knowledge Foundation.
+
+It is open and licensed under the GNU Affero General Public License (AGPL) v3.0
+whose full text may be found at:
+
+http://www.fsf.org/licensing/licenses/agpl-3.0.html
+

bin/canada.py

@@ -0,0 +1,103 @@
+'''
+Script to sort out the tags imported from ca.ckan.net to thedatahub.org and
+got mangled in the process.
+'''
+
+import re
+from optparse import OptionParser
+import copy
+
+import ckanclient
+from status import Status
+
+def sort_out_tags(source_ckan_uri,
+                  dest_ckan_uri, dest_api_key,
+                  ):
+    ckan1 = ckanclient.CkanClient(base_location=source_ckan_uri)
+    ckan2 = ckanclient.CkanClient(base_location=dest_ckan_uri,
+                                  api_key=dest_api_key)
+
+    # ensure group exists
+    group = 'country-ca'
+    assert group in set(ckan2.group_register_get())
+    group_to_change = 'canadagov'
+
+    # work out tag mappings
+    tag_status = Status('tag mapping')
+    tag_replace_map = {}
+    source_tags = ckan1.tag_register_get()
+    for tag in source_tags:
+        mangled_tag = re.sub('[-._]', '', tag)
+        replacement_tag = tag
+        # Change underscores to hyphens
+        replacement_tag = replacement_tag.replace('_', '-')
+        # Remove trailing punctuation
+        if replacement_tag[-1] in '_-.':
+            replacement_tag = replacement_tag[:-1]
+        if replacement_tag[0] in '_-.':
+            replacement_tag = replacement_tag[1:]
+        if mangled_tag == replacement_tag:
+            tag_status.record('Unchanged', mangled_tag, do_print=False)
+            continue
+        if mangled_tag in tag_replace_map and tag_replace_map[mangled_tag] != replacement_tag:
+            print 'Warning - can\'t differentiate %s : %s / %s' % \
+                  (mangled_tag, tag_replace_map[mangled_tag], replacement_tag)
+        tag_status.record('Mapping added', '%s:%s' % (mangled_tag, replacement_tag), do_print=False)
+        tag_replace_map[mangled_tag] = replacement_tag
+    example_map = tag_replace_map.items()[0]
+    print tag_status
+
+    # Custom mappings
+    tag_replace_map['metaimportedfromcackannet'] = 'meta.imported-from-ca-ckan-net'
+
+    # edit packages
+    pkg_status = Status('Packages')
+    pkgs = ckan2.group_entity_get(group)['packages']
+    print 'Packages in the group: %i' % len(pkgs)
+    for pkg_name in pkgs:
+        pkg = ckan2.package_entity_get(pkg_name)
+        original_pkg = copy.deepcopy(pkg)
+
+        # Change tags
+        edited_tags = [tag_replace_map.get(tag, tag) for tag in pkg['tags']]
+        if 'canada' in edited_tags:
+            edited_tags.remove('canada')
+
+        if group_to_change in pkg['groups']:
+            pkg['groups'].remove(group_to_change)
+            edited_tags.append('canada-gov')
+
+        if set(pkg['tags']) != set(edited_tags):
+            pkg['tags'] = edited_tags
+            print '%s: %r -> %r' % (pkg_name, sorted(original_pkg['tags']), sorted(edited_tags))
+
+        if pkg == original_pkg:
+            pkg_status.record('Unchanged', pkg_name)
+            continue
+
+        try:
+            ckan2.package_entity_put(pkg)
+        except ckanclient.CkanApiError, e:
+            pkg_status.record('Error: %r' % e.args, pkg_name)
+            continue
+
+        pkg_status.record('Successfully changed', pkg_name)
+
+    print pkg_status
+
+usage = '''%prog [OPTIONS] <source_ckan_api_uri> <destination_ckan_api_uri>
+Recopy tags that got mangled in Canadian copy.'''
+parser = OptionParser(usage=usage)
+parser.add_option("-k", "--destination-ckan-api-key", dest="destination_ckan_api_key",
+                  help="Destination CKAN's API key", metavar="API-KEY")
+
+(options, args) = parser.parse_args()
+
+assert len(args) == 2, 'The source and destination CKAN API URIs are the only two arguments. Found: %r' % args
+source_ckan_uri, destination_ckan_uri = args
+print 'Key: ', options.destination_ckan_api_key
+
+sort_out_tags(source_ckan_uri,
+              destination_ckan_uri,
+              options.destination_ckan_api_key,
+)

bin/status.py

@@ -0,0 +1,26 @@
+from collections import defaultdict
+
+class Status:
+    '''When looping through objects and doing operations to them,
+    this is a useful object to keep track of what happens and
+    summarise the numbers at the end.'''
+    def __init__(self, obj_type_str=None):
+        self.obj_type_str = obj_type_str
+        self.pkg_status = defaultdict(list) # reason: [pkgs]
+
+    def record(self, status_category, pkg_name, do_print=True):
+        self.pkg_status[status_category].append(pkg_name)
+        if do_print:
+            print '%s: %s' % (pkg_name, status_category)
+
+    def __str__(self):
+        status = '\nStatus'
+        if self.obj_type_str:
+            status += ' of: %s' % self.obj_type_str
+        status += '\n'
+        status += '\n'.join([ \
+            '%s: %i (e.g. %s)' % (category, len(pkg_names), sorted(pkg_names)[0]) \
+            for (category, pkg_names) in self.pkg_status.items()])
+        status += '\nTotal: %i\n' % sum([len(pkg_names) for pkg_names in self.pkg_status.values()])
+        return status
+

build.sh

@@ -15,11 +15,25 @@
 # Over time, as CKAN and its dependencies follow conventions more closely the
 # edge cases handled by this script should go away and the build should become
 # simpler.
+#
+# Install:
+#
+#     sudo apt-get update
+#     sudo apt-get install -y wget
+#     echo "deb http://apt.3aims.com/buildkit-0.2.2 lucid universe" | sudo tee /etc/apt/sources.list.d/3aims.list
+#     wget -qO- "http://apt.3aims.com/packages_public.key" | sudo apt-key add -
+#     sudo apt-get update
+#     sudo apt-get install buildkit-deb buildkit-apt-repo
+
 
-CKAN_PACKAGE_VERSION=08
-DEPS_PACKAGE_VERSION=05
-CKAN_PATH=/home/james/Documents/Work/OKFN/code/pyenv/src/ckan
-BUILDKIT_REPO_PATH="/var/lib/buildkit/repo/ckan-1.5"
+CKAN_PACKAGE_VERSION=$1
+DEPS_PACKAGE_VERSION=$2
+# If you don't run this command from the CKAN source directory, specify the 
+# path to CKAN here
+CKAN_PATH=$PWD
+# You'll need to create the repo if it doesn't exist:
+# sudo -u buildkit buildkit repo clone base_lucid ckan-1.5.1
+REPO_NAME="ckan-1.5.1"
 PIP_DOWNLOAD_CACHE=${CKAN_PATH}/build/env/cache
 EMAIL=packaging@okfn.org
 NAME="James Gardner"
@@ -54,8 +68,7 @@ cp ${CKAN_PATH}/build/buildkit/env/build/licenses/dist/buildkit/*.deb ${CKAN_PAT
 echo "done."
 
 # Add the .debs to the repository and the export the latest files for upload
-echo "Adding the packages to the $BUILDKIT_REPO_PATH repo ..."
-sudo -u buildkit buildkit repo remove -a $BUILDKIT_REPO_PATH
-sudo -u buildkit buildkit repo add $BUILDKIT_REPO_PATH ${CKAN_PATH}/dist/buildkit/*.deb
-echo "done."
-
+# echo "Adding the packages to the $REPO_NAME repo using files in ${CKAN_PATH}/dist/buildkit/*.deb ..."
+# sudo -u buildkit buildkit repo remove -a $REPO_NAME dummy_arg
+# sudo -u buildkit buildkit repo add $REPO_NAME ${CKAN_PATH}/dist/buildkit/*.deb
+# echo "done."

ckan/__init__.py

@@ -1,17 +1,18 @@
 __version__ = '1.5.2a'
 __description__ = 'Comprehensive Knowledge Archive Network (CKAN) Software'
 __long_description__ = \
-'''The CKAN software is used to run the Comprehensive Knowledge Archive
-Network (CKAN) site: http://www.ckan.net.
+'''CKAN software provides a hub for datasets. The flagship site running CKAN 
+is theDataHub.org but it is also used for dozens of other open data websites
+run by governments, agencies and citizens.
 
-The Comprehensive Knowledge Archive Network is a registry of open
-knowledge packages and projects (and a few closed ones). CKAN is the
-place to search for open knowledge resources as well as register your
-own - be that a set of Shakespeare's works, a global population density
-database, the voting records of MPs, or 30 years of US patents.
+CKAN provides a place to search for open knowledge resources as well as 
+register your own - be that a set of Shakespeare's works, a global 
+population density database, the voting records of MPs, or 30 years of 
+US patents.
 
-Those familiar with freshmeat or CPAN can think of CKAN as providing an
-analogous service for open knowledge. 
+CKAN is an abbreviation for 'Comprehensive Knowledge Archive Network'. 
+Those familiar with Freshmeat or CPAN can think of CKAN as providing an
+analogous service for open data and knowledge. 
 '''
 __license__ = 'AGPL'
 

ckan/authz.py

@@ -47,6 +47,7 @@ def is_authorized(cls, username, action, domain_object):
         if isinstance(username, str):
             username = username.decode('utf8')
         assert isinstance(username, unicode), type(username)
+
         for extension in cls.extensions:
             authorized = extension.is_authorized(username,
                                                  action,

ckan/config/deployment.ini_tmpl

@@ -24,7 +24,7 @@ app_instance_uuid = ${app_instance_uuid}
 
 # List the names of CKAN extensions to activate.
 # Note: This line is required to be here for packaging, even if it is empty.
-ckan.plugins = 
+ckan.plugins = stats
 
 # If you'd like to fine-tune the individual locations of the cache data dirs
 # for the Cache data, or the Session saves, un-comment the desired settings
@@ -109,7 +109,7 @@ ckan.site_description =
 ckan.site_url =
 
 ## Favicon (default is the CKAN software favicon)
-ckan.favicon = http://assets.okfn.org/p/ckan/img/ckan.ico
+ckan.favicon = /images/icons/ckan.ico
 
 ## Solr support
 #solr_url = http://127.0.0.1:8983/solr
@@ -160,6 +160,13 @@ ckan.locale_default = en
 ckan.locale_order = en de fr it es pl ru nl sv no cs_CZ hu pt_BR fi bg ca sq sr sr_Latn
 ckan.locales_filtered_out = el ro lt sl
 
+## ===================================
+## Extensions
+
+## Config option to enable the (1 day) cache for stats
+## Default (if not defined) is True as stats computations are intensive
+# ckanext.stats.cache_enabled = True
+
 # Logging configuration
 [loggers]
 keys = root, ckan, ckanext

ckan/config/environment.py

@@ -81,6 +81,10 @@ def find_controller(self, controller):
                           'ckan.site_id for SOLR search-index rebuild to work.'
         config['ckan.site_id'] = ckan_host
 
+    # Check if SOLR schema is compatible
+    from ckan.lib.search import check_solr_schema_version
+    check_solr_schema_version()
+
     config['routes.map'] = make_map()
     config['pylons.app_globals'] = app_globals.Globals()
     config['pylons.h'] = ckan.lib.helpers

ckan/config/middleware.py

@@ -75,6 +75,14 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
 
     if asbool(config.get('openid_enabled', 'true')):
         from repoze.who.plugins.openid.identification import OpenIdIdentificationPlugin
+        # Monkey patches for repoze.who.openid
+        # Fixes #1659 - enable log-out when CKAN mounted at non-root URL 
+        from ckan.lib import repoze_patch
+        OpenIdIdentificationPlugin.identify = repoze_patch.identify
+        OpenIdIdentificationPlugin.redirect_to_logged_in = repoze_patch.redirect_to_logged_in
+        OpenIdIdentificationPlugin._redirect_to_loginform = repoze_patch._redirect_to_loginform
+        OpenIdIdentificationPlugin.challenge = repoze_patch.challenge
+
         who_parser.identifiers = [i for i in who_parser.identifiers if \
                 not isinstance(i, OpenIdIdentificationPlugin)]
         who_parser.challengers = [i for i in who_parser.challengers if \