Merge conflicts

.tx/config

@@ -1,7 +1,7 @@
 [main]
 host = http://www.transifex.net
 
-[ckan.1-7]
+[ckan.1-8]
 file_filter = ckan/i18n/<lang>/LC_MESSAGES/ckan.po
 source_file = ckan/i18n/ckan.pot
 source_lang = en

CHANGELOG.txt

@@ -9,17 +9,37 @@ v1.8
   Some of CKAN's dependencies have been updated and some removed.
 * Requirements have been updated see doc/install-from-source.rst
   users will need to do a new pip install (#2592)
-* [#2304,#2305] New 'follow' feature. You'll now see a 'Followers' tab on user
-  and dataset pages, where you can see how many users are following that user
-  or dataset.  If you're logged in, you'll see a 'Follow' button on the pages
-  of datasets and other users that you can click to follow them. Also when
-  logged in, if you go to your own user page you'll see a new 'Dashboard' tab
-  where you can see an activity stream from of all the users and datasets that
-  you're following. There are also API calls for the follow features, see the
-  Action API reference documentation.
+* [#2304] New 'follow' feature. You'll now see a 'Followers' tab on user and
+  dataset pages, where you can see how many users are following that user or
+  dataset.  If you're logged in, you'll see a 'Follow' button on the pages of
+  datasets and other users that you can click to follow them. There are also
+  API calls for the follow features, see the Action API reference
+  documentation.
+* [#2305] New user dashboards, implemented by Sven R. Kunze
+  (https://github.com/kunsv) as part of his Masters thesis. When logged in, if
+  you go to your own user page you'll see a new 'Dashboard' tab where you can
+  see an activity stream from of all the users and datasets that you're
+  following.
 * [#2345] New action API reference docs. The documentation for CKAN's Action
   API has been rewritten, with each function and its arguments and return
   values now individually documented.
+* Template helper functions are now restricted by default. This is
+  part of a process of refactoring the CKAN code. By default only
+  those helper functions listed in lib.helpers.__allowed_functions__
+  are available to templates. The full functions can still be made
+  available by `setting ckan.restrict_template_vars = false` in your
+  .ini config file. Only restricted functions will be allowed in
+  future versions of CKAN.
+* [#2842] Allow sort ordering of dataset listings on group pages
+
+API changes and deprecation:
+* [#2313] Deprecated functions related to the old faceting data structure have
+  been removed:  `helpers.py:facet_items()`, `facets.html:facet_sidebar()`,
+  `facets.html:facet_list_items()`.
+  Internal use of the old facets datastructure (attached to the context,
+  `c.facets`) has been superseded by use of the improved facet data structure,
+  `c.search_facets`.  The old data structure is still available on `c.facets`,
+  but is deprecated, and will be removed in v1.9.
 
 v1.7.1 2012-06-20
 =================

LICENSE.txt

@@ -22,9 +22,9 @@ Note
 ====
 
 CKAN is sometimes packaged directly with other software (listed in
-requires/lucid_conflict.txt). In these cases, we are required to list the
-licenses of the packaged softare too. They are all AGPL compatible and read as
-follows in the next sections.
+pip-requirements.txt, pip-requirements-test.txt and pip-requirements-docs.txt).
+In these cases, we are required to list the licenses of the packaged softare
+too. They are all AGPL compatible and read as follows in the next sections.
 
 WebHelpers
 ----------

ckan/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '1.8a'
+__version__ = '1.8b'
 __description__ = 'Comprehensive Knowledge Archive Network (CKAN) Software'
 __long_description__ = \
 '''CKAN software provides a hub for datasets. The flagship site running CKAN 

ckan/config/deployment.ini_tmpl

@@ -24,7 +24,7 @@ app_instance_uuid = ${app_instance_uuid}
 
 # List the names of CKAN extensions to activate.
 # Note: This line is required to be here for packaging, even if it is empty.
-ckan.plugins = stats synchronous_search
+ckan.plugins = stats
 
 # If you'd like to fine-tune the individual locations of the cache data dirs
 # for the Cache data, or the Session saves, un-comment the desired settings
@@ -112,6 +112,11 @@ ckan.gravatar_default = identicon
 ## Solr support
 #solr_url = http://127.0.0.1:8983/solr
 
+## Automatic indexing. Make all changes immediately available via the search
+## after editing or creating a dataset. Default is true. If for some reason
+## you need the indexing to occur asynchronously, set this option to 0.
+# ckan.search.automatic_indexing = 1
+
 ## An 'id' for the site (using, for example, when creating entries in a common search index) 
 ## If not specified derived from the site_url
 # ckan.site_id = ckan.net
@@ -141,9 +146,9 @@ ckan.dump_dir = %(here)s/dump
 ckan.backup_dir = %(here)s/backup
 
 # Default authorizations for new domain objects
-#ckan.default_roles.Package = {"visitor": ["editor"], "logged_in": ["editor"]}
+#ckan.default_roles.Package = {"visitor": ["reader"], "logged_in": ["reader"]}
 #ckan.default_roles.Group = {"visitor": ["reader"], "logged_in": ["reader"]}
-#ckan.default_roles.System = {"visitor": ["anon_editor"], "logged_in": ["editor"]}
+#ckan.default_roles.System = {"visitor": ["reader"], "logged_in": ["editor"]}
 #ckan.default_roles.AuthorizationGroup = {"visitor": ["reader"], "logged_in": ["reader"]}
 
 ## Ckan public and private recaptcha keys [localhost]
@@ -191,6 +196,34 @@ ckan.feeds.author_name =
 # If not set, then the value in `ckan.site_url` is used.
 ckan.feeds.author_link =
 
+## File Store
+#
+# CKAN allows users to upload files directly to file storage either on the local
+# file system or to online ‘cloud’ storage like Amazon S3 or Google Storage.
+#
+# If you are using local file storage, remember to set ckan.site_url.
+#
+# To enable cloud storage (Google or S3), first run: pip install boto
+#
+# @see http://docs.ckan.org/en/latest/filestore.html
+
+# 'Bucket' to use for file storage
+#ckan.storage.bucket = my-bucket-name
+
+# To enable local file storage:
+#ofs.impl = pairtree
+#ofs.storage_dir = /my/path/to/storage/root/directory
+
+# To enable Google cloud storage:
+#ofs.impl = google
+#ofs.gs_access_key_id =
+#ofs.gs_secret_access_key =
+
+# To enable S3 cloud storage:
+#ofs.impl = s3
+#ofs.aws_access_key_id = ....
+#ofs.aws_secret_access_key = ....
+
 ## Webstore
 ## Uncommment to enable datastore
 # ckan.datastore.enabled = 1

ckan/config/environment.py

@@ -16,21 +16,22 @@
 import ckan.model as model
 import ckan.plugins as p
 import ckan.lib.helpers as h
-import ckan.lib.search as search
 import ckan.lib.app_globals as app_globals
 
+log = logging.getLogger(__name__)
 
 # Suppress benign warning 'Unbuilt egg for setuptools'
 warnings.simplefilter('ignore', UserWarning)
 
+
 class _Helpers(object):
     ''' Helper object giving access to template helpers stopping
     missing functions from causing template exceptions. Useful if
     templates have helper functions provided by extensions that have
     not been enabled. '''
     def __init__(self, helpers, restrict=True):
         functions = {}
-        allowed = helpers.__allowed_functions__
+        allowed = helpers.__allowed_functions__[:]
         # list of functions due to be deprecated
         self.deprecated = []
 
@@ -40,8 +41,14 @@ def __init__(self, helpers, restrict=True):
                 if restrict:
                     continue
             functions[helper] = getattr(helpers, helper)
+            if helper in allowed:
+                allowed.remove(helper)
         self.functions = functions
 
+        if allowed:
+            raise Exception('Template helper function(s) `%s` not defined'
+                            % ', '.join(allowed))
+
         # extend helper functions with ones supplied by plugins
         extra_helpers = []
         for plugin in p.PluginImplementations(p.ITemplateHelpers):
@@ -93,13 +100,16 @@ def load_environment(global_conf, app_conf):
     from pylons.wsgiapp import PylonsApp
     import pkg_resources
     find_controller_generic = PylonsApp.find_controller
+
     # This is from pylons 1.0 source, will monkey-patch into 0.9.7
     def find_controller(self, controller):
         if controller in self.controller_classes:
             return self.controller_classes[controller]
         # Check to see if its a dotted name
         if '.' in controller or ':' in controller:
-            mycontroller = pkg_resources.EntryPoint.parse('x=%s' % controller).load(False)
+            mycontroller = pkg_resources \
+                .EntryPoint \
+                .parse('x=%s' % controller).load(False)
             self.controller_classes[controller] = mycontroller
             return mycontroller
         return find_controller_generic(self, controller)
@@ -122,6 +132,13 @@ def find_controller(self, controller):
     # load all CKAN plugins
     p.load_all(config)
 
+    # Load the synchronous search plugin, unless already loaded or
+    # explicitly disabled
+    if not 'synchronous_search' in config.get('ckan.plugins',[]) and \
+            asbool(config.get('ckan.search.automatic_indexing', True)):
+        log.debug('Loading the synchronous search plugin')
+        p.load('synchronous_search')
+
     for plugin in p.PluginImplementations(p.IConfigurer):
         # must do update in place as this does not work:
         # config = plugin.update_config(config)
@@ -143,6 +160,9 @@ def find_controller(self, controller):
 
     # Init SOLR settings and check if the schema is compatible
     #from ckan.lib.search import SolrSettings, check_solr_schema_version
+
+    # lib.search is imported here as we need the config enabled and parsed
+    import ckan.lib.search as search
     search.SolrSettings.init(config.get('solr_url'),
                              config.get('solr_user'),
                              config.get('solr_password'))
@@ -152,11 +172,13 @@ def find_controller(self, controller):
     config['pylons.app_globals'] = app_globals.Globals()
 
     # add helper functions
-    restrict_helpers = asbool(config.get('ckan.restrict_template_vars', 'true'))
+    restrict_helpers = asbool(
+        config.get('ckan.restrict_template_vars', 'true'))
     helpers = _Helpers(h, restrict_helpers)
     config['pylons.h'] = helpers
 
-    ## redo template setup to use genshi.search_path (so remove std template setup)
+    # Redo template setup to use genshi.search_path
+    # (so remove std template setup)
     template_paths = [paths['templates'][0]]
     extra_template_paths = config.get('extra_template_paths', '')
     if extra_template_paths:
@@ -165,6 +187,7 @@ def find_controller(self, controller):
 
     # Translator (i18n)
     translator = Translator(pylons.translator)
+
     def template_loaded(template):
         translator.setup(template)
 
@@ -195,8 +218,6 @@ def template_loaded(template):
     #                                                               #
     #################################################################
 
-
-
     '''
     This code is based on Genshi code
 
@@ -265,11 +286,14 @@ def genshi_lookup_attr(cls, obj, key):
 
     # Setup the SQLAlchemy database engine
     # Suppress a couple of sqlalchemy warnings
-    warnings.filterwarnings('ignore', '^Unicode type received non-unicode bind param value', sqlalchemy.exc.SAWarning)
-    warnings.filterwarnings('ignore', "^Did not recognize type 'BIGINT' of column 'size'", sqlalchemy.exc.SAWarning)
-    warnings.filterwarnings('ignore', "^Did not recognize type 'tsvector' of column 'search_vector'", sqlalchemy.exc.SAWarning)
+    msgs = ['^Unicode type received non-unicode bind param value',
+            "^Did not recognize type 'BIGINT' of column 'size'",
+            "^Did not recognize type 'tsvector' of column 'search_vector'"
+            ]
+    for msg in msgs:
+        warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning)
 
-    ckan_db = os.environ.get('CKAN_DB') 
+    ckan_db = os.environ.get('CKAN_DB')
 
     if ckan_db:
         config['sqlalchemy.url'] = ckan_db
@@ -288,4 +312,3 @@ def genshi_lookup_attr(cls, obj, key):
 
     for plugin in p.PluginImplementations(p.IConfigurable):
         plugin.configure(config)
-

ckan/config/middleware.py

@@ -20,7 +20,7 @@
 
 from ckan.plugins import PluginImplementations
 from ckan.plugins.interfaces import IMiddleware
-from ckan.lib.i18n import get_locales
+from ckan.lib.i18n import get_locales_from_config
 
 from ckan.config.environment import load_environment
 
@@ -145,7 +145,7 @@ class I18nMiddleware(object):
     def __init__(self, app, config):
         self.app = app
         self.default_locale = config.get('ckan.locale_default', 'en')
-        self.local_list = get_locales()
+        self.local_list = get_locales_from_config()
 
     def __call__(self, environ, start_response):
         # strip the language selector from the requested url

ckan/config/routing.py

@@ -159,7 +159,8 @@ def make_map():
 
     with SubMapper(map, controller='related') as m:
         m.connect('related_list', '/dataset/{id}/related', action='list')
-        m.connect('related_read', '/dataset/{id}/related/{related_id}', action='read')
+        m.connect('related_read', '/apps/{id}', action='read')
+        m.connect('related_dashboard', '/apps', action='dashboard')
 
     with SubMapper(map, controller='package') as m:
         m.connect('/dataset', action='search')

ckan/controllers/datastore.py

@@ -1,9 +1,9 @@
 from ckan.lib.base import BaseController, abort, _, c, response, request, g
 import ckan.model as model
-from ckan.lib.helpers import json
 from ckan.lib.jsonp import jsonpify
 from ckan.logic import get_action, check_access
-from ckan.logic import NotFound, NotAuthorized, ValidationError
+from ckan.logic import NotFound, NotAuthorized
+
 
 
 class DatastoreController(BaseController):
@@ -21,8 +21,6 @@ def read(self, id, url=''):
 
         try:
             resource = get_action('resource_show')(context, {'id': id})
-            if not resource.get('webstore_url', ''):
-                return {'error': 'DataStore is disabled for this resource'}
             self._make_redirect(id, url)
             return ''
         except NotFound:
@@ -35,13 +33,12 @@ def write(self, id, url):
         context = {'model': model, 'session': model.Session,
                    'user': c.user or c.author}
         try:
-            resource = model.Resource.get(id)
-            if not resource:
-                abort(404, _('Resource not found'))
-            if not resource.webstore_url:
-                return {'error': 'DataStore is disabled for this resource'}
-            context["resource"] = resource
             check_access('resource_update', context, {'id': id})
+            resource_dict = get_action('resource_show')(context,{'id':id})
+            if not resource_dict['webstore_url']:
+                resource_dict['webstore_url'] = u'active'
+                get_action('resource_update')(context,resource_dict)
+
             self._make_redirect(id, url)
             return ''
         except NotFound:

ckan/controllers/feed.py

@@ -432,7 +432,7 @@ def _parse_url_params(self):
         """
 
         try:
-            page = int(request.params.get('page', 1))
+            page = int(request.params.get('page', 1)) or 1
         except ValueError:
             abort(400, ('"page" parameter must be an integer'))