Merge branch 'master' into 1852-default-resource-views

Conflicts:
	ckan/new_tests/logic/auth/test_create.py

CHANGELOG.rst

@@ -29,6 +29,17 @@ API changes and deprecations
   Note that logic.get_action() and toolkit.get_action() are *not* deprecated,
   core code and plugin code should still use ``get_action()``.
 
+* Cross-Origin Resource Sharing (CORS) support is no longer enabled by
+  default. Previously, Access-Control-Allow-* response headers were added for
+  all requests, with Access-Control-Allow-Origin set to the wildcard value
+  ``*``. To re-enable CORS, use the new ``ckan.cors`` configuration settings
+  (:ref:`ckan.cors.origin_allow_all` and :ref:`ckan.cors.origin_whitelist`).
+
+* The HttpOnly flag will be set on the authorization cookie by default. For
+  enhanced security, we recommend using the HttpOnly flag, but this behaviour
+  can be changed in the ``Repoze.who`` settings detailed in the Config File
+  Options documentation (:ref:`who.httponly`).
+
 Template changes
 ----------------
 

ckan/config/deployment.ini_tmpl

@@ -78,6 +78,15 @@ ckan.site_id = default
 #ckan.simple_search = 1
 
 
+## CORS Settings
+
+# If cors.origin_allow_all is true, all origins are allowed.
+# If false, the cors.origin_whitelist is used.
+# ckan.cors.origin_allow_all = true
+# cors.origin_whitelist is a space separated list of allowed domains.
+# ckan.cors.origin_whitelist = http://example1.com http://example2.com
+
+
 ## Plugins Settings
 
 # Note: Add ``datastore`` to enable the CKAN DataStore

ckan/config/environment.py

@@ -290,6 +290,10 @@ def update_config():
     if asbool(config.get('ckan.legacy_templates', 'no')):
         # We want the new template path for extra snippets like the
         # dataviewer and also for some testing stuff
+        msg = 'Support for Genshi templates is deprecated and will be removed'\
+            ' in a future release'
+        log.warn(msg)
+
         template_paths = [legacy_templates_path, jinja2_templates_path]
     else:
         template_paths = [jinja2_templates_path, legacy_templates_path]

ckan/config/middleware.py

@@ -18,7 +18,6 @@
 from routes.middleware import RoutesMiddleware
 from repoze.who.config import WhoConfig
 from repoze.who.middleware import PluggableAuthenticationMiddleware
-from repoze.who.plugins.auth_tkt import make_plugin as auth_tkt_make_plugin
 from fanstatic import Fanstatic
 
 from ckan.plugins import PluginImplementations
@@ -192,11 +191,6 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf):
 
     return app
 
-def ckan_auth_tkt_make_app(**kw):
-    if not len(kw.get('secret', '')) or kw.get('secret') == 'somesecret':
-        kw['secret'] = config['beaker.session.secret']
-    return auth_tkt_make_plugin(**kw)
-
 
 class I18nMiddleware(object):
     """I18n Middleware selects the language based on the url

ckan/config/routing.py

@@ -282,10 +282,6 @@ def make_map():
     map.redirect('/groups', '/group')
     map.redirect('/groups/{url:.*}', '/group/{url}')
 
-    ##to get back formalchemy uncomment these lines
-    ##map.connect('/group/new', controller='group_formalchemy', action='new')
-    ##map.connect('/group/edit/{id}', controller='group_formalchemy', action='edit')
-
     # These named routes are used for custom group forms which will use the
     # names below based on the group.type ('group' is the default type)
     with SubMapper(map, controller='group') as m:

ckan/config/solr/schema.xml

@@ -97,6 +97,7 @@
 
     <field name="capacity" type="string" indexed="true" stored="true" multiValued="false"/>
 
+    <field name="res_name" type="textgen" indexed="true" stored="true" multiValued="true" />
     <field name="res_description" type="textgen" indexed="true" stored="true" multiValued="true"/>
     <field name="res_format" type="string" indexed="true" stored="true" multiValued="true"/>
     <field name="res_url" type="string" indexed="true" stored="true" multiValued="true"/>
@@ -161,8 +162,9 @@
 <copyField source="notes" dest="text"/>
 <copyField source="tags" dest="text"/>
 <copyField source="groups" dest="text"/>
+<copyField source="res_name" dest="text"/>
 <copyField source="res_description" dest="text"/>
 <copyField source="maintainer" dest="text"/>
 <copyField source="author" dest="text"/>
 
-</schema>
+</schema>

ckan/config/who.ini

@@ -1,5 +1,5 @@
 [plugin:auth_tkt]
-use = ckan.config.middleware:ckan_auth_tkt_make_app
+use = ckan.lib.auth_tkt:make_plugin
 # If no secret key is defined here, beaker.session.secret will be used
 #secret = somesecret
 
@@ -49,7 +49,8 @@ plugins =
     auth_tkt
 
 [authenticators]
-plugins = 
+plugins =
+    auth_tkt
     ckan.lib.authenticator:OpenIDAuthenticator
     ckan.lib.authenticator:UsernamePasswordAuthenticator
 

ckan/controllers/group.py

@@ -884,39 +884,3 @@ def _get_group_dict(self, id):
             abort(404, _('Group not found'))
         except NotAuthorized:
             abort(401, _('Unauthorized to read group %s') % id)
-
-    def _render_edit_form(self, fs):
-        # errors arrive in c.error and fs.errors
-        c.fieldset = fs
-        return render('group/edit_form.html')
-
-    def _update(self, fs, group_name, group_id):
-        '''
-        Writes the POST data (associated with a group edit) to the database
-        @input c.error
-        '''
-        validation = fs.validate()
-        if not validation:
-            c.form = self._render_edit_form(fs)
-            raise base.ValidationException(fs)
-
-        try:
-            fs.sync()
-        except Exception, inst:
-            model.Session.rollback()
-            raise
-        else:
-            model.Session.commit()
-
-    def _update_authz(self, fs):
-        validation = fs.validate()
-        if not validation:
-            c.form = self._render_edit_form(fs)
-            raise base.ValidationException(fs)
-        try:
-            fs.sync()
-        except Exception, inst:
-            model.Session.rollback()
-            raise
-        else:
-            model.Session.commit()

ckan/controllers/home.py

@@ -111,7 +111,7 @@ def index(self):
             if msg:
                 h.flash_notice(msg, allow_html=True)
 
-        # START OF DIRTYNESS
+        # START OF DIRTINESS
         def get_group(id):
             def _get_group_type(id):
                 """
@@ -178,7 +178,7 @@ def db_to_form_schema(group_type=None):
 
         c.group_package_stuff = dirty_cached_group_stuff
 
-        # END OF DIRTYNESS
+        # END OF DIRTINESS
         return base.render('home/index.html', cache_force=True)
 
     def license(self):

ckan/controllers/package.py

@@ -1,7 +1,6 @@
 import logging
 from urllib import urlencode
 import datetime
-import os
 import mimetypes
 import cgi
 
@@ -14,7 +13,6 @@
 import ckan.logic as logic
 import ckan.lib.base as base
 import ckan.lib.maintain as maintain
-import ckan.lib.package_saver as package_saver
 import ckan.lib.i18n as i18n
 import ckan.lib.navl.dictization_functions as dict_fns
 import ckan.lib.accept as accept
@@ -313,10 +311,10 @@ def pager_url(q=None, page=None):
                       extra_vars={'dataset_type': package_type})
 
     def _content_type_from_extension(self, ext):
-        ct, mu, ext = accept.parse_extension(ext)
+        ct, ext = accept.parse_extension(ext)
         if not ct:
-            return None, None, None,
-        return ct, ext, (NewTextTemplate, MarkupTemplate)[mu]
+            return None, None
+        return ct, ext
 
     def _content_type_from_accept(self):
         """
@@ -325,8 +323,8 @@ def _content_type_from_accept(self):
         it accurately.  TextTemplate must be used for non-xml templates
         whilst all that are some sort of XML should use MarkupTemplate.
         """
-        ct, mu, ext = accept.parse_header(request.headers.get('Accept', ''))
-        return ct, ext, (NewTextTemplate, MarkupTemplate)[mu]
+        ct, ext = accept.parse_header(request.headers.get('Accept', ''))
+        return ct, ext
 
     def resources(self, id):
         package_type = self._get_package_type(id.split('@')[0])
@@ -358,16 +356,15 @@ def resources(self, id):
 
     def read(self, id, format='html'):
         if not format == 'html':
-            ctype, extension, loader = \
+            ctype, extension = \
                 self._content_type_from_extension(format)
             if not ctype:
                 # An unknown format, we'll carry on in case it is a
                 # revision specifier and re-constitute the original id
                 id = "%s.%s" % (id, format)
-                ctype, format, loader = "text/html; charset=utf-8", "html", \
-                    MarkupTemplate
+                ctype, format = "text/html; charset=utf-8", "html"
         else:
-            ctype, format, loader = self._content_type_from_accept()
+            ctype, format = self._content_type_from_accept()
 
         response.headers['Content-Type'] = ctype
 
@@ -421,13 +418,11 @@ def read(self, id, format='html'):
         self._setup_template_variables(context, {'id': id},
                                        package_type=package_type)
 
-        package_saver.PackageSaver().render_package(c.pkg_dict, context)
-
         template = self._read_template(package_type)
         template = template[:template.index('.') + 1] + format
 
         try:
-            return render(template, loader_class=loader,
+            return render(template,
                           extra_vars={'dataset_type': package_type})
         except ckan.lib.render.TemplateNotFound:
             msg = _("Viewing {package_type} datasets in {format} format is "
@@ -840,7 +835,6 @@ def edit(self, id, data=None, errors=None, error_summary=None):
                                   'dataset_type': package_type})
 
     def read_ajax(self, id, revision=None):
-        package_type = self._get_package_type(id)
         context = {'model': model, 'session': model.Session,
                    'user': c.user or c.author, 'auth_user_obj': c.userobj,
                    'revision_id': revision}
@@ -1053,16 +1047,6 @@ def _form_save_redirect(self, pkgname, action, package_type=None):
                 url = h.url_for('{0}_read'.format(package_type), id=pkgname)
         redirect(url)
 
-    def _adjust_license_id_options(self, pkg, fs):
-        options = fs.license_id.render_opts['options']
-        is_included = False
-        for option in options:
-            license_id = option[1]
-            if license_id == pkg.license_id:
-                is_included = True
-        if not is_included:
-            options.insert(1, (pkg.license_id, pkg.license_id))
-
     def delete(self, id):
 
         if 'cancel' in request.params:

ckan/lib/accept.py

@@ -9,10 +9,10 @@
 accept_re = re.compile("^(?P<ct>[^;]+)[ \t]*(;[ \t]*q=(?P<q>[0-9.]+)){0,1}$")
 
 accept_types = {
-    #   Name         : ContentType,             Is Markup?, Extension
-    "text/html": ("text/html; charset=utf-8",  True,  'html'),
-    "text/n3": ("text/n3; charset=utf-8",    False, 'n3'),
-    "application/rdf+xml": ("application/rdf+xml; charset=utf-8", True, 'rdf'),
+    #   Name         : ContentType,                                Extension
+    "text/html": ("text/html; charset=utf-8",                     'html'),
+    "text/n3": ("text/n3; charset=utf-8",                         'n3'),
+    "application/rdf+xml": ("application/rdf+xml; charset=utf-8", 'rdf'),
 }
 accept_by_extension = {
     "rdf": "application/rdf+xml",
@@ -28,7 +28,7 @@ def parse_extension(file_ext):
     ext = accept_by_extension.get(file_ext, None)
     if ext:
         return accept_types[ext]
-    return (None, None, None,)
+    return (None, None)
 
 
 def parse_header(accept_header=''):

ckan/lib/auth_tkt.py

@@ -0,0 +1,78 @@
+import os
+
+from pylons import config
+from repoze.who.plugins import auth_tkt as repoze_auth_tkt
+
+_bool = repoze_auth_tkt._bool
+
+import logging
+log = logging.getLogger(__name__)
+
+
+class CkanAuthTktCookiePlugin(repoze_auth_tkt.AuthTktCookiePlugin):
+
+    def __init__(self, httponly, *args, **kwargs):
+        super(CkanAuthTktCookiePlugin, self).__init__(*args, **kwargs)
+        self.httponly = httponly
+
+    def _get_cookies(self, *args, **kwargs):
+        '''
+        Override method in superclass to ensure HttpOnly is set appropriately.
+        '''
+        super_cookies = super(CkanAuthTktCookiePlugin, self). \
+            _get_cookies(*args, **kwargs)
+
+        cookies = []
+        for k, v in super_cookies:
+            replace_with = '; HttpOnly' if self.httponly else ''
+            v = v.replace('; HttpOnly', '') + replace_with
+            cookies.append((k, v))
+
+        return cookies
+
+
+def make_plugin(secret=None,
+                secretfile=None,
+                cookie_name='auth_tkt',
+                secure=False,
+                include_ip=False,
+                timeout=None,
+                reissue_time=None,
+                userid_checker=None):
+    from repoze.who.utils import resolveDotted
+
+    # ckan specific: get secret from beaker setting if necessary
+    if secret is None or secret == 'somesecret':
+        secret = config['beaker.session.secret']
+
+    # Set httponly based on config value. Default is True
+    httponly = config.get('who.httponly', True)
+
+    # Set secure based on config value. Default is False
+    secure = config.get('who.secure', False)
+
+    # back to repoze boilerplate
+    if (secret is None and secretfile is None):
+        raise ValueError("One of 'secret' or 'secretfile' must not be None.")
+    if (secret is not None and secretfile is not None):
+        raise ValueError("Specify only one of 'secret' or 'secretfile'.")
+    if secretfile:
+        secretfile = os.path.abspath(os.path.expanduser(secretfile))
+        if not os.path.exists(secretfile):
+            raise ValueError("No such 'secretfile': %s" % secretfile)
+        secret = open(secretfile).read().strip()
+    if timeout:
+        timeout = int(timeout)
+    if reissue_time:
+        reissue_time = int(reissue_time)
+    if userid_checker is not None:
+        userid_checker = resolveDotted(userid_checker)
+    plugin = CkanAuthTktCookiePlugin(_bool(httponly),
+                                     secret,
+                                     cookie_name,
+                                     _bool(secure),
+                                     _bool(include_ip),
+                                     timeout,
+                                     reissue_time,
+                                     userid_checker)
+    return plugin