diff --git a/README.rst b/README.rst index 17fce2e20d0..93d80e668cc 100644 --- a/README.rst +++ b/README.rst @@ -18,14 +18,21 @@ Installation See the `CKAN Documentation `_ for installation instructions. -Community ---------- +Support +------- -* Developer mailing list: `ckan-dev@lists.okfn.org `_ -* Developer IRC channel: `#ckan on irc.freenode.net `_ -* `Issue tracker `_ -* `CKAN tag on StackOverflow `_ -* `Wiki `_ +If you need help with CKAN or want to ask a question about CKAN, use either the +`ckan-discuss`_ mailing list or the `CKAN tag on Stack Overflow`_ (try +searching the Stack Overflow and ckan-discuss archives for an answer to your +question first). + +If you've found a bug in CKAN, open a new issue on CKAN's `GitHub Issues`_ (try +searching first to see if there's already an issue for your bug). + + +.. _CKAN tag on Stack Overflow: http://stackoverflow.com/questions/tagged/ckan +.. _ckan-discuss: http://lists.okfn.org/mailman/listinfo/ckan-discuss +.. _GitHub Issues: https://github.com/okfn/ckan/issues Contributing to CKAN @@ -34,11 +41,22 @@ Contributing to CKAN For contributing to CKAN or its documentation, see `CONTRIBUTING `_. +If you want to talk about CKAN development say hi to the CKAN developers on the +`ckan-dev`_ mailing list or in the `#ckan`_ IRC channel on irc.freenode.net. + +If you've figured out how to do something with CKAN and want to document it for +others, make a new page on the `CKAN wiki`_, and tell us about it on +`ckan-dev`_. + +.. _ckan-dev: http://lists.okfn.org/mailman/listinfo/ckan-dev +.. _#ckan: http://webchat.freenode.net/?channels=ckan +.. _CKAN Wiki: https://github.com/okfn/ckan/wiki + Copying and License ------------------- -This material is copyright (c) 2006-2011 Open Knowledge Foundation. +This material is copyright (c) 2006-2013 Open Knowledge Foundation. It is open and licensed under the GNU Affero General Public License (AGPL) v3.0 whose full text may be found at: diff --git a/ckan/config/environment.py b/ckan/config/environment.py index 1b0151e11b1..897c3321cb6 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -9,7 +9,6 @@ from paste.deploy.converters import asbool import sqlalchemy from pylons import config -from pylons.i18n import _, ungettext from genshi.template import TemplateLoader from genshi.filters.i18n import Translator @@ -18,10 +17,12 @@ import ckan.plugins as p import ckan.lib.helpers as h import ckan.lib.app_globals as app_globals +import ckan.lib.jinja_extensions as jinja_extensions + +from ckan.common import _, ungettext log = logging.getLogger(__name__) -import lib.jinja_extensions # Suppress benign warning 'Unbuilt egg for setuptools' warnings.simplefilter('ignore', UserWarning) @@ -297,22 +298,22 @@ def genshi_lookup_attr(cls, obj, key): # Create Jinja2 environment - env = lib.jinja_extensions.Environment( - loader=lib.jinja_extensions.CkanFileSystemLoader(template_paths), + env = jinja_extensions.Environment( + loader=jinja_extensions.CkanFileSystemLoader(template_paths), autoescape=True, extensions=['jinja2.ext.do', 'jinja2.ext.with_', - lib.jinja_extensions.SnippetExtension, - lib.jinja_extensions.CkanExtend, - lib.jinja_extensions.CkanInternationalizationExtension, - lib.jinja_extensions.LinkForExtension, - lib.jinja_extensions.ResourceExtension, - lib.jinja_extensions.UrlForStaticExtension, - lib.jinja_extensions.UrlForExtension] + jinja_extensions.SnippetExtension, + jinja_extensions.CkanExtend, + jinja_extensions.CkanInternationalizationExtension, + jinja_extensions.LinkForExtension, + jinja_extensions.ResourceExtension, + jinja_extensions.UrlForStaticExtension, + jinja_extensions.UrlForExtension] ) env.install_gettext_callables(_, ungettext, newstyle=True) # custom filters - env.filters['empty_and_escape'] = lib.jinja_extensions.empty_and_escape - env.filters['truncate'] = lib.jinja_extensions.truncate + env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape + env.filters['truncate'] = jinja_extensions.truncate config['pylons.app_globals'].jinja_env = env # CONFIGURATION OPTIONS HERE (note: all config options will override diff --git a/ckan/controllers/api.py b/ckan/controllers/api.py index ee1f34f6ebd..30dc0beb8a5 100644 --- a/ckan/controllers/api.py +++ b/ckan/controllers/api.py @@ -5,12 +5,9 @@ import glob import urllib -from pylons import c, request, response -from pylons.i18n import _, gettext -from paste.util.multidict import MultiDict from webob.multidict import UnicodeMultiDict +from paste.util.multidict import MultiDict -import ckan.rating import ckan.model as model import ckan.logic as logic import ckan.lib.base as base @@ -20,6 +17,8 @@ import ckan.lib.jsonp as jsonp import ckan.lib.munge as munge +from ckan.common import _, c, request, response + log = logging.getLogger(__name__) @@ -159,7 +158,7 @@ def action(self, logic_function, ver=None): except KeyError: log.error('Can\'t find logic function: %s' % logic_function) return self._finish_bad_request( - gettext('Action name not known: %s') % str(logic_function)) + _('Action name not known: %s') % str(logic_function)) context = {'model': model, 'session': model.Session, 'user': c.user, 'api_version': ver} @@ -172,12 +171,12 @@ def action(self, logic_function, ver=None): except ValueError, inst: log.error('Bad request data: %s' % str(inst)) return self._finish_bad_request( - gettext('JSON Error: %s') % str(inst)) + _('JSON Error: %s') % str(inst)) if not isinstance(request_data, dict): # this occurs if request_data is blank log.error('Bad request data - not dict: %r' % request_data) return self._finish_bad_request( - gettext('Bad request data: %s') % + _('Bad request data: %s') % 'Request data JSON decoded to %r but ' 'it needs to be a dictionary.' % request_data) try: @@ -265,7 +264,7 @@ def list(self, ver=None, register=None, subregister=None, id=None): action = self._get_action_from_map(action_map, register, subregister) if not action: return self._finish_bad_request( - gettext('Cannot list entity of this type: %s') % register) + _('Cannot list entity of this type: %s') % register) try: return self._finish_ok(action(context, {'id': id})) except NotFound, e: @@ -296,7 +295,7 @@ def show(self, ver=None, register=None, subregister=None, action = self._get_action_from_map(action_map, register, subregister) if not action: return self._finish_bad_request( - gettext('Cannot read entity of this type: %s') % register) + _('Cannot read entity of this type: %s') % register) try: return self._finish_ok(action(context, data_dict)) except NotFound, e: @@ -331,12 +330,12 @@ def create(self, ver=None, register=None, subregister=None, data_dict.update(request_data) except ValueError, inst: return self._finish_bad_request( - gettext('JSON Error: %s') % str(inst)) + _('JSON Error: %s') % str(inst)) action = self._get_action_from_map(action_map, register, subregister) if not action: return self._finish_bad_request( - gettext('Cannot create new entity of this type: %s %s') % + _('Cannot create new entity of this type: %s %s') % (register, subregister)) try: @@ -390,12 +389,12 @@ def update(self, ver=None, register=None, subregister=None, data_dict.update(request_data) except ValueError, inst: return self._finish_bad_request( - gettext('JSON Error: %s') % str(inst)) + _('JSON Error: %s') % str(inst)) action = self._get_action_from_map(action_map, register, subregister) if not action: return self._finish_bad_request( - gettext('Cannot update entity of this type: %s') % + _('Cannot update entity of this type: %s') % register.encode('utf-8')) try: response_data = action(context, data_dict) @@ -439,7 +438,7 @@ def delete(self, ver=None, register=None, subregister=None, action = self._get_action_from_map(action_map, register, subregister) if not action: return self._finish_bad_request( - gettext('Cannot delete entity of this type: %s %s') % + _('Cannot delete entity of this type: %s %s') % (register, subregister or '')) try: response_data = action(context, data_dict) @@ -462,11 +461,11 @@ def search(self, ver=None, register=None): id = request.params['since_id'] if not id: return self._finish_bad_request( - gettext(u'No revision specified')) + _(u'No revision specified')) rev = model.Session.query(model.Revision).get(id) if rev is None: return self._finish_not_found( - gettext(u'There is no revision with id: %s') % id) + _(u'There is no revision with id: %s') % id) since_time = rev.timestamp elif 'since_time' in request.params: since_time_str = request.params['since_time'] @@ -476,7 +475,7 @@ def search(self, ver=None, register=None): return self._finish_bad_request('ValueError: %s' % inst) else: return self._finish_bad_request( - gettext("Missing search term ('since_id=UUID' or " + + _("Missing search term ('since_id=UUID' or " + " 'since_time=TIMESTAMP')")) revs = model.Session.query(model.Revision).\ filter(model.Revision.timestamp > since_time) @@ -486,7 +485,7 @@ def search(self, ver=None, register=None): params = MultiDict(self._get_search_params(request.params)) except ValueError, e: return self._finish_bad_request( - gettext('Could not read parameters: %r' % e)) + _('Could not read parameters: %r' % e)) # if using API v2, default to returning the package ID if # no field list is specified @@ -543,10 +542,10 @@ def search(self, ver=None, register=None): except search.SearchError, e: log.exception(e) return self._finish_bad_request( - gettext('Bad search option: %s') % e) + _('Bad search option: %s') % e) else: return self._finish_not_found( - gettext('Unknown register: %s') % register) + _('Unknown register: %s') % register) @classmethod def _get_search_params(cls, request_params): @@ -555,7 +554,7 @@ def _get_search_params(cls, request_params): qjson_param = request_params['qjson'].replace('\\\\u', '\\u') params = h.json.loads(qjson_param, encoding='utf8') except ValueError, e: - raise ValueError(gettext('Malformed qjson value') + ': %r' + raise ValueError(_('Malformed qjson value: %r') % e) elif len(request_params) == 1 and \ len(request_params.values()[0]) < 2 and \ @@ -572,7 +571,7 @@ def _get_search_params(cls, request_params): def markdown(self, ver=None): raw_markdown = request.params.get('q', '') - results = ckan.misc.MarkdownFormat().to_html(raw_markdown) + results = h.render_markdown(raw_markdown) return self._finish_ok(results) diff --git a/ckan/controllers/feed.py b/ckan/controllers/feed.py index e1ea58fa511..c22aa1080c7 100644 --- a/ckan/controllers/feed.py +++ b/ckan/controllers/feed.py @@ -21,16 +21,17 @@ # TODO fix imports import logging import urlparse +from urllib import urlencode import webhelpers.feedgenerator from pylons import config -from pylons.i18n import _ -from urllib import urlencode -from ckan import model -from ckan.lib.base import BaseController, c, request, response, json, abort, g +import ckan.model as model +import ckan.lib.base as base import ckan.lib.helpers as h -from ckan.logic import get_action, NotFound +import ckan.logic as logic + +from ckan.common import _, g, c, request, response, json # TODO make the item list configurable ITEMS_LIMIT = 20 @@ -55,7 +56,7 @@ def _package_search(data_dict): data_dict['rows'] = ITEMS_LIMIT # package_search action modifies the data_dict, so keep our copy intact. - query = get_action('package_search')(context, data_dict.copy()) + query = logic.get_action('package_search')(context, data_dict.copy()) return query['count'], query['results'] @@ -151,7 +152,7 @@ def _create_atom_id(resource_path, authority_name=None, date_string=None): return ':'.join(['tag', tagging_entity, resource_path]) -class FeedController(BaseController): +class FeedController(base.BaseController): base_url = config.get('ckan.site_url') def _alternate_url(self, params, **kwargs): @@ -170,9 +171,9 @@ def group(self, id): try: context = {'model': model, 'session': model.Session, 'user': c.user or c.author} - group_dict = get_action('group_show')(context, {'id': id}) - except NotFound: - abort(404, _('Group not found')) + group_dict = logic.get_action('group_show')(context, {'id': id}) + except logic.NotFound: + base.abort(404, _('Group not found')) data_dict, params = self._parse_url_params() data_dict['fq'] = 'groups:"%s"' % id @@ -281,9 +282,9 @@ def custom(self): try: page = int(request.params.get('page', 1)) except ValueError: - abort(400, _('"page" parameter must be a positive integer')) + base.abort(400, _('"page" parameter must be a positive integer')) if page < 0: - abort(400, _('"page" parameter must be a positive integer')) + base.abort(400, _('"page" parameter must be a positive integer')) limit = ITEMS_LIMIT data_dict = { @@ -433,9 +434,9 @@ def _parse_url_params(self): try: page = int(request.params.get('page', 1)) or 1 except ValueError: - abort(400, _('"page" parameter must be a positive integer')) + base.abort(400, _('"page" parameter must be a positive integer')) if page < 0: - abort(400, _('"page" parameter must be a positive integer')) + base.abort(400, _('"page" parameter must be a positive integer')) limit = ITEMS_LIMIT data_dict = { diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index 9adf26af9aa..140dd1d4eba 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -201,14 +201,7 @@ def _read(self, id, limit): else: q += ' groups:"%s"' % c.group_dict.get('name') - try: - description_formatted = ckan.misc.MarkdownFormat().to_html( - c.group_dict.get('description', '')) - c.description_formatted = genshi.HTML(description_formatted) - except Exception, e: - error_msg = "%s" %\ - _("Cannot render description") - c.description_formatted = genshi.HTML(error_msg) + c.description_formatted = h.render_markdown(c.group_dict.get('description')) context['return_query'] = True diff --git a/ckan/controllers/home.py b/ckan/controllers/home.py index 7a9fe062a0d..fa5fddf3d9f 100644 --- a/ckan/controllers/home.py +++ b/ckan/controllers/home.py @@ -1,5 +1,4 @@ -from pylons.i18n import _ -from pylons import g, c, config, cache +from pylons import config, cache import sqlalchemy.exc import ckan.logic as logic @@ -9,6 +8,8 @@ import ckan.model as model import ckan.lib.helpers as h +from ckan.common import _, g, c + CACHE_PARAMETERS = ['__cache', '__no_cache__'] # horrible hack @@ -143,7 +144,7 @@ def db_to_form_schema(group_type=None): global dirty_cached_group_stuff if not dirty_cached_group_stuff: groups_data = [] - groups = config.get('demo.featured_groups', '').split() + groups = config.get('ckan.featured_groups', '').split() for group_name in groups: group = get_group(group_name) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 2be958f72ed..3d726e67ad2 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -174,7 +174,6 @@ def _sort_by(fields): else: c.sort_by_fields = [field.split()[0] for field in sort_by.split(',')] - c.sort_by_selected = sort_by def pager_url(q=None, page=None): params = list(params_nopage) @@ -217,10 +216,13 @@ def pager_url(q=None, page=None): facets = OrderedDict() - default_facet_titles = {'groups': _('Groups'), - 'tags': _('Tags'), - 'res_format': _('Formats'), - 'license': _('Licence'), } + default_facet_titles = { + 'organization': _('Organizations'), + 'groups': _('Groups'), + 'tags': _('Tags'), + 'res_format': _('Formats'), + 'license': _('Licence'), + } for facet in g.facets: if facet in default_facet_titles: @@ -245,6 +247,7 @@ def pager_url(q=None, page=None): } query = get_action('package_search')(context, data_dict) + c.sort_by_selected = query['sort'] c.page = h.Page( collection=query['results'], @@ -607,9 +610,16 @@ def new_resource(self, id, data=None, errors=None, error_summary=None): abort(401, _('Unauthorized to update dataset')) if not len(data_dict['resources']): # no data so keep on page - h.flash_error(_('You must add at least one data resource')) - redirect(h.url_for(controller='package', - action='new_resource', id=id)) + msg = _('You must add at least one data resource') + # On new templates do not use flash message + if g.legacy_templates: + h.flash_error(msg) + redirect(h.url_for(controller='package', + action='new_resource', id=id)) + else: + errors = {} + error_summary = {_('Error'): msg} + return self.new_resource(id, data, errors, error_summary) # we have a resource so let them add metadata redirect(h.url_for(controller='package', action='new_metadata', id=id)) diff --git a/ckan/controllers/related.py b/ckan/controllers/related.py index 68a68c8c1d1..690f82faee2 100644 --- a/ckan/controllers/related.py +++ b/ckan/controllers/related.py @@ -1,15 +1,14 @@ +import urllib + import ckan.model as model import ckan.logic as logic import ckan.lib.base as base import ckan.lib.helpers as h import ckan.lib.navl.dictization_functions as df -import pylons.i18n as i18n +from ckan.common import _, c -_ = i18n._ -import urllib -c = base.c abort = base.abort _get_action = logic.get_action diff --git a/ckan/controllers/revision.py b/ckan/controllers/revision.py index 32809602cc3..bbfb2c4b610 100644 --- a/ckan/controllers/revision.py +++ b/ckan/controllers/revision.py @@ -1,14 +1,14 @@ from datetime import datetime, timedelta -from pylons.i18n import get_lang, _ -from pylons import c, request - -from ckan.logic import NotAuthorized, check_access +from pylons.i18n import get_lang +import ckan.logic as logic import ckan.lib.base as base import ckan.model as model import ckan.lib.helpers as h +from ckan.common import _, c, request + class RevisionController(base.BaseController): @@ -18,15 +18,15 @@ def __before__(self, action, **env): context = {'model': model, 'user': c.user or c.author} if c.user: try: - check_access('revision_change_state', context) + logic.check_access('revision_change_state', context) c.revision_change_state_allowed = True - except NotAuthorized: + except logic.NotAuthorized: c.revision_change_state_allowed = False else: c.revision_change_state_allowed = False try: - check_access('site_read', context) - except NotAuthorized: + logic.check_access('site_read', context) + except logic.NotAuthorized: base.abort(401, _('Not authorized to see this page')) def index(self): diff --git a/ckan/controllers/tag.py b/ckan/controllers/tag.py index 699d31fceb3..631a8523f98 100644 --- a/ckan/controllers/tag.py +++ b/ckan/controllers/tag.py @@ -1,11 +1,12 @@ -from pylons.i18n import _ -from pylons import request, c, config +from pylons import config import ckan.logic as logic import ckan.model as model import ckan.lib.base as base import ckan.lib.helpers as h +from ckan.common import _, request, c + LIMIT = 25 diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index d79d68a64e8..10cb0e9a78d 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -1,13 +1,10 @@ import logging from urllib import quote -from pylons import session, c, g, request, config -from pylons.i18n import _ -import genshi +from pylons import config import ckan.lib.i18n as i18n import ckan.lib.base as base -import ckan.misc as misc import ckan.model as model import ckan.lib.helpers as h import ckan.new_authz as new_authz @@ -17,6 +14,8 @@ import ckan.lib.mailer as mailer import ckan.lib.navl.dictization_functions as dictization_functions +from ckan.common import _, session, c, g, request + log = logging.getLogger(__name__) @@ -72,7 +71,7 @@ def _setup_template_variables(self, context, data_dict): abort(401, _('Not authorized to see this page')) c.user_dict = user_dict c.is_myself = user_dict['name'] == c.user - c.about_formatted = self._format_about(user_dict['about']) + c.about_formatted = h.render_markdown(user_dict['about']) ## end hooks @@ -574,8 +573,8 @@ def dashboard(self, id=None, offset=0): context, {'id': c.userobj.id, 'q': q}) c.dashboard_activity_stream_context = self._get_dashboard_context( filter_type, filter_id, q) - c.dashboard_activity_stream = h.dashboard_activity_stream( - id, filter_type, filter_id, offset) + c.dashboard_activity_stream = h.dashboard_activity_stream(filter_type, + filter_id, offset) # Mark the user's new activities as old whenever they view their # dashboard page. @@ -621,13 +620,3 @@ def unfollow(self, id): or e.error_dict) h.flash_error(error_message) h.redirect_to(controller='user', action='read', id=id) - - def _format_about(self, about): - about_formatted = misc.MarkdownFormat().to_html(about) - try: - html = genshi.HTML(about_formatted) - except genshi.ParseError, e: - log.error('Could not print "about" field Field: %r Error: %r', - about, e) - html = _('Error: Could not parse About text') - return html diff --git a/ckan/lib/activity_streams.py b/ckan/lib/activity_streams.py index fa1a7f04e87..6c4908a5ff2 100644 --- a/ckan/lib/activity_streams.py +++ b/ckan/lib/activity_streams.py @@ -1,13 +1,13 @@ import re -import datetime -from pylons.i18n import _ from webhelpers.html import literal import ckan.lib.helpers as h import ckan.lib.base as base import ckan.logic as logic +from ckan.common import _ + # get_snippet_*() functions replace placeholders like {user}, {dataset}, etc. # in activity strings with HTML representations of particular users, datasets, # etc. diff --git a/ckan/lib/alphabet_paginate.py b/ckan/lib/alphabet_paginate.py index b75bf27e83b..5bffd70594b 100644 --- a/ckan/lib/alphabet_paginate.py +++ b/ckan/lib/alphabet_paginate.py @@ -15,12 +15,13 @@ ''' from itertools import dropwhile import re + from sqlalchemy import __version__ as sqav from sqlalchemy.orm.query import Query -from pylons.i18n import _ from webhelpers.html.builder import HTML from routes import url_for + class AlphaPage(object): def __init__(self, collection, alpha_attribute, page, other_text, paging_threshold=50, controller_name='tag'): diff --git a/ckan/lib/app_globals.py b/ckan/lib/app_globals.py index e98f975ca7d..6a687dba365 100644 --- a/ckan/lib/app_globals.py +++ b/ckan/lib/app_globals.py @@ -47,7 +47,7 @@ 'ckan.api_url': {}, # split string - 'search.facets': {'default': 'groups tags res_format license', + 'search.facets': {'default': 'organization groups tags res_format license', 'type': 'split', 'name': 'facets'}, 'package_hide_extras': {'type': 'split'}, @@ -57,6 +57,8 @@ 'openid_enabled': {'default': 'true', 'type' : 'bool'}, 'debug': {'default': 'false', 'type' : 'bool'}, 'ckan.debug_supress_header' : {'default': 'false', 'type' : 'bool'}, + 'ckan.legacy_templates' : {'default': 'false', 'type' : 'bool'}, + 'ckan.tracking_enabled' : {'default': 'false', 'type' : 'bool'}, # int 'ckan.datasets_per_page': {'default': '20', 'type': 'int'}, diff --git a/ckan/lib/base.py b/ckan/lib/base.py index 7f944a3393b..c72f09f09db 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -6,12 +6,12 @@ import time from paste.deploy.converters import asbool -from pylons import c, cache, config, g, request, response, session +from pylons import cache, config, session from pylons.controllers import WSGIController from pylons.controllers.util import abort as _abort from pylons.controllers.util import redirect_to, redirect from pylons.decorators import jsonify, validate -from pylons.i18n import _, ungettext, N_, gettext, ngettext +from pylons.i18n import N_, gettext, ngettext from pylons.templating import cached_template, pylons_globals from genshi.template import MarkupTemplate from genshi.template.text import NewTextTemplate @@ -25,7 +25,11 @@ import ckan.lib.app_globals as app_globals from ckan.plugins import PluginImplementations, IGenshiStreamFilter import ckan.model as model -from ckan.common import json + +# These imports are for legacy usages and will be removed soon these should +# be imported directly from ckan.common for internal ckan code and via the +# plugins.toolkit for extensions. +from ckan.common import json, _, ungettext, c, g, request, response log = logging.getLogger(__name__) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 1fc8de336df..e76a0ddf444 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -99,21 +99,21 @@ def _setup_app(self): class ManageDb(CkanCommand): '''Perform various tasks on the database. - db create # alias of db upgrade - db init # create and put in default data + db create - alias of db upgrade + db init - create and put in default data db clean - db upgrade [{version no.}] # Data migrate - db version # returns current version of data schema - db dump {file-path} # dump to a pg_dump file - db dump-rdf {dataset-name} {file-path} - db simple-dump-csv {file-path} # dump just datasets in CSV format - db simple-dump-json {file-path} # dump just datasets in JSON format - db user-dump-csv {file-path} # dump user information to a CSV file - db send-rdf {talis-store} {username} {password} - db load {file-path} # load a pg_dump from a file - db load-only {file-path} # load a pg_dump from a file but don\'t do - # the schema upgrade or search indexing - db create-from-model # create database from the model (indexes not made) + db upgrade [version no.] - Data migrate + db version - returns current version of data schema + db dump FILE_PATH - dump to a pg_dump file + db dump-rdf DATASET_NAME FILE_PATH + db simple-dump-csv FILE_PATH - dump just datasets in CSV format + db simple-dump-json FILE_PATH - dump just datasets in JSON format + db user-dump-csv FILE_PATH - dump user information to a CSV file + db send-rdf TALIS_STORE USERNAME PASSWORD + db load FILE_PATH - load a pg_dump from a file + db load-only FILE_PATH - load a pg_dump from a file but don\'t do + the schema upgrade or search indexing + db create-from-model - create database from the model (indexes not made) ''' summary = __doc__.split('\n')[0] usage = __doc__ @@ -312,10 +312,12 @@ class SearchIndexCommand(CkanCommand): '''Creates a search index for all datasets Usage: - search-index [-i] [-o] [-r] [-e] rebuild [dataset-name] - reindex dataset-name if given, if not then rebuild full search index (all datasets) - search-index check - checks for datasets not indexed - search-index show {dataset-name} - shows index of a dataset - search-index clear [dataset-name] - clears the search index for the provided dataset or for the whole ckan instance + search-index [-i] [-o] [-r] [-e] rebuild [dataset_name] - reindex dataset_name if given, if not then rebuild + full search index (all datasets) + search-index check - checks for datasets not indexed + search-index show DATASET_NAME - shows index of a dataset + search-index clear [dataset_name] - clears the search index for the provided dataset or + for the whole ckan instance ''' summary = __doc__.split('\n')[0] @@ -434,7 +436,7 @@ def command(self): class RDFExport(CkanCommand): - ''' + '''Export active datasets as RDF This command dumps out all currently active datasets as RDF into the specified folder. @@ -498,8 +500,8 @@ class Sysadmin(CkanCommand): Usage: sysadmin - lists sysadmins sysadmin list - lists sysadmins - sysadmin add - add a user as a sysadmin - sysadmin remove - removes user from sysadmins + sysadmin add USERNAME - add a user as a sysadmin + sysadmin remove USERNAME - removes user from sysadmins ''' summary = __doc__.split('\n')[0] @@ -579,16 +581,16 @@ class UserCmd(CkanCommand): Usage: user - lists users user list - lists users - user - shows user properties - user add [=] + user USERNAME - shows user properties + user add USERNAME [FIELD1=VALUE1 FIELD2=VALUE2 ...] - add a user (prompts for password if not supplied). Field can be: apikey password email - user setpass - set user password (prompts) - user remove - removes user from users - user search - searches for a user name + user setpass USERNAME - set user password (prompts) + user remove USERNAME - removes user from users + user search QUERY - searches for a user name ''' summary = __doc__.split('\n')[0] usage = __doc__ @@ -735,11 +737,11 @@ class DatasetCmd(CkanCommand): '''Manage datasets Usage: - dataset - shows dataset properties - dataset show - shows dataset properties + dataset DATASET_NAME|ID - shows dataset properties + dataset show DATASET_NAME|ID - shows dataset properties dataset list - lists datasets - dataset delete - changes dataset state to 'deleted' - dataset purge - removes dataset from db entirely + dataset delete [DATASET_NAME|ID] - changes dataset state to 'deleted' + dataset purge [DATASET_NAME|ID] - removes dataset from db entirely ''' summary = __doc__.split('\n')[0] usage = __doc__ @@ -816,12 +818,11 @@ class Celery(CkanCommand): '''Celery daemon Usage: - celeryd - run the celery daemon - celeryd run - run the celery daemon - celeryd run concurrency - run the celery daemon with - argument 'concurrency' - celeryd view - view all tasks in the queue - celeryd clean - delete all tasks in the queue + celeryd - run the celery daemon + celeryd run concurrency - run the celery daemon with + argument 'concurrency' + celeryd view - view all tasks in the queue + celeryd clean - delete all tasks in the queue ''' min_args = 0 max_args = 2 @@ -940,8 +941,8 @@ class Tracking(CkanCommand): '''Update tracking statistics Usage: - tracking update [start-date] - update tracking stats - tracking export [start-date] - export tracking stats to a csv file + tracking update [start_date] - update tracking stats + tracking export FILE [start_date] - export tracking stats to a csv file ''' summary = __doc__.split('\n')[0] @@ -1047,7 +1048,7 @@ def export_tracking(self, engine, output_filename): for r in total_views]) def update_tracking(self, engine, summary_date): - PACKAGE_URL = '/dataset/' + PACKAGE_URL = '%/dataset/' # clear out existing data before adding new sql = '''DELETE FROM tracking_summary WHERE tracking_date='%s'; ''' % summary_date @@ -1073,7 +1074,7 @@ def update_tracking(self, engine, summary_date): sql = '''UPDATE tracking_summary t SET package_id = COALESCE( (SELECT id FROM package p - WHERE t.url = %s || p.name) + WHERE t.url LIKE %s || p.name) ,'~~not~found~~') WHERE t.package_id IS NULL AND tracking_type = 'page';''' @@ -1116,7 +1117,7 @@ def update_tracking(self, engine, summary_date): engine.execute(sql) class PluginInfo(CkanCommand): - ''' Provide info on installed plugins. + '''Provide info on installed plugins. ''' summary = __doc__.split('\n')[0] @@ -1220,8 +1221,8 @@ class CreateTestDataCommand(CkanCommand): create-test-data user - create a user 'tester' with api key 'tester' create-test-data translations - annakarenina, warandpeace, and some test translations of terms - create-test-data vocabs - annakerenina, warandpeace, and some test - vocabularies + create-test-data vocabs - annakerenina, warandpeace, and some test + vocabularies ''' summary = __doc__.split('\n')[0] @@ -1271,7 +1272,7 @@ class Profile(CkanCommand): by runsnakerun. Usage: - profile {url} + profile URL e.g. profile /data/search @@ -1328,15 +1329,15 @@ def profile_url(url): class CreateColorSchemeCommand(CkanCommand): - ''' Create or remove a color scheme. + '''Create or remove a color scheme. - less will need to generate the css files after this has been run + After running this, you'll need to regenerate the css files. See paster's less command for details. - color - creates a random color scheme - color clear - clears any color scheme - color '' - uses as base color eg '#ff00ff' must be quoted. - color - a float between 0.0 and 1.0 used as base hue - color - html color name used for base color eg lightblue + color - creates a random color scheme + color clear - clears any color scheme + color <'HEX'> - uses as base color eg '#ff00ff' must be quoted. + color - a float between 0.0 and 1.0 used as base hue + color - html color name used for base color eg lightblue ''' summary = __doc__.split('\n')[0] @@ -1587,8 +1588,8 @@ def command(self): class TranslationsCommand(CkanCommand): '''Translation helper functions - trans js - generate the javascript translations - trans mangle - mangle the zh_TW translations for testing + trans js - generate the javascript translations + trans mangle - mangle the zh_TW translations for testing ''' summary = __doc__.split('\n')[0] @@ -1744,7 +1745,7 @@ class MinifyCommand(CkanCommand): Usage: - paster minify [--clean] + paster minify [--clean] PATH for example: @@ -1919,7 +1920,7 @@ def compile_less(self, root, less_bin, color): class FrontEndBuildCommand(CkanCommand): - ''' Creates and minifies css and JavaScript files + '''Creates and minifies css and JavaScript files Usage: diff --git a/ckan/lib/create_test_data.py b/ckan/lib/create_test_data.py index e2aba3b242f..3edf865c7c7 100644 --- a/ckan/lib/create_test_data.py +++ b/ckan/lib/create_test_data.py @@ -148,15 +148,15 @@ def create_arbitrary(cls, package_dicts, relationships=[], new_group_names = set() new_groups = {} - rev = model.repo.new_revision() - rev.author = cls.author - rev.message = u'Creating test packages.' admins_list = defaultdict(list) # package_name: admin_names if package_dicts: if isinstance(package_dicts, dict): package_dicts = [package_dicts] for item in package_dicts: + rev = model.repo.new_revision() + rev.author = cls.author + rev.message = u'Creating test packages.' pkg_dict = {} for field in cls.pkg_core_fields: if item.has_key(field): @@ -245,7 +245,7 @@ def create_arbitrary(cls, package_dicts, relationships=[], model.setup_default_user_roles(pkg, admins=[]) for admin in admins: admins_list[item['name']].append(admin) - model.repo.commit_and_remove() + model.repo.commit_and_remove() needs_commit = False @@ -464,7 +464,7 @@ def create(cls, auth_profile="", package_type=None): model.Session.add_all([ model.User(name=u'tester', apikey=u'tester', password=u'tester'), model.User(name=u'joeadmin', password=u'joeadmin'), - model.User(name=u'annafan', about=u'I love reading Annakarenina. My site: anna.com', password=u'annafan'), + model.User(name=u'annafan', about=u'I love reading Annakarenina. My site: http://anna.com', password=u'annafan'), model.User(name=u'russianfan', password=u'russianfan'), sysadmin, ]) diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index d61c68e73b1..d2a74071923 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -4,7 +4,6 @@ from pylons import config from sqlalchemy.sql import select -import ckan.misc as misc import ckan.logic as logic import ckan.plugins as plugins import ckan.lib.helpers as h @@ -530,7 +529,7 @@ def package_to_api(pkg, context): dictized['license'] = pkg.license.title if pkg.license else None dictized['ratings_average'] = pkg.get_average_rating() dictized['ratings_count'] = len(pkg.ratings) - dictized['notes_rendered'] = misc.MarkdownFormat().to_html(pkg.notes) + dictized['notes_rendered'] = h.render_markdown(pkg.notes) site_url = config.get('ckan.site_url', None) if site_url: diff --git a/ckan/lib/email_notifications.py b/ckan/lib/email_notifications.py index 20ae39b6ebf..9b12fcdd082 100644 --- a/ckan/lib/email_notifications.py +++ b/ckan/lib/email_notifications.py @@ -8,12 +8,13 @@ import re import pylons -import pylons.i18n import ckan.model as model import ckan.logic as logic import ckan.lib.base as base +from ckan.common import ungettext + def string_to_timedelta(s): '''Parse a string s and return a standard datetime.timedelta object. @@ -96,7 +97,7 @@ def _notifications_for_activities(activities, user_dict): # say something about the contents of the activities, or single out # certain types of activity to be sent in their own individual emails, # etc. - subject = pylons.i18n.ungettext( + subject = ungettext( "1 new activity from {site_title}", "{n} new activities from {site_title}", len(activities)).format( diff --git a/ckan/lib/formatters.py b/ckan/lib/formatters.py index 9c8b656987a..5e3d51f7c6d 100644 --- a/ckan/lib/formatters.py +++ b/ckan/lib/formatters.py @@ -1,10 +1,11 @@ import datetime -from pylons.i18n import _, ungettext from babel import numbers import ckan.lib.i18n as i18n +from ckan.common import _, ungettext + ################################################## # # diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index ff15eac6a53..04960d6c94c 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -636,7 +636,7 @@ def markdown_extract(text, extract_length=190): will not be truncated.''' if (text is None) or (text.strip() == ''): return '' - plain = re.sub(r'<.*?>', '', markdown(text)) + plain = RE_MD_HTML_TAGS.sub('', markdown(text)) if not extract_length or len(plain) < extract_length: return literal(plain) return literal(unicode(truncate(plain, length=extract_length, indicator='...', whole_word=True))) @@ -956,7 +956,7 @@ def related_item_link(related_item_dict): def tag_link(tag): url = url_for(controller='tag', action='read', id=tag['name']) - return link_to(tag['name'], url) + return link_to(tag.get('title', tag['name']), url) def group_link(group): @@ -1289,7 +1289,7 @@ def user_in_org_or_group(group_id): def dashboard_activity_stream(user_id, filter_type=None, filter_id=None, offset=0): - '''Return the dashboard activity stream of the given user. + '''Return the dashboard activity stream of the current user. :param user_id: the id of the user :type user_id: string @@ -1317,7 +1317,7 @@ def dashboard_activity_stream(user_id, filter_type=None, filter_id=None, return action_function(context, {'id': filter_id, 'offset': offset}) else: return logic.get_action('dashboard_activity_list_html')( - context, {'id': user_id, 'offset': offset}) + context, {'offset': offset}) def recently_changed_packages_activity_stream(): @@ -1367,12 +1367,77 @@ def get_request_param(parameter_name, default=None): return request.params.get(parameter_name, default) -def render_markdown(data): +# find all inner text of html eg `moo` gets `moo` but not of tags +# as this would lead to linkifying links if they are urls. +RE_MD_GET_INNER_HTML = re.compile( + r'(^|(?:<(?!a\b)[^>]*>))([^<]+)(?=<|$)', + flags=re.UNICODE +) + +# find all `internal links` eg. tag:moo, dataset:1234, tag:"my tag" +RE_MD_INTERNAL_LINK = re.compile( + r'\b(tag|package|dataset|group):((")?(?(3)[ \w\-.]+|[\w\-.]+)(?(3)"))', + flags=re.UNICODE +) + +# find external links eg http://foo.com, https://bar.org/foobar.html +RE_MD_EXTERNAL_LINK = re.compile( + r'(\bhttps?:\/\/[\w\-\.,@?^=%&;:\/~\\+#]*)', + flags=re.UNICODE +) + +# find all tags but ignore < in the strings so that we can use it correctly +# in markdown +RE_MD_HTML_TAGS = re.compile('<[^><]*>') + + +def html_auto_link(data): + '''Linkifies HTML + + tag:... converted to a tag link + dataset:... converted to a dataset link + group:... converted to a group link + http://... converted to a link + ''' + + LINK_FNS = { + 'tag': tag_link, + 'group': group_link, + 'dataset': dataset_link, + 'package': dataset_link, + } + + def makelink(matchobj): + obj = matchobj.group(1) + name = matchobj.group(2) + title = '%s:%s' % (obj, name) + return LINK_FNS[obj]({'name': name.strip('"'), 'title': title}) + + def link(matchobj): + return '%s' \ + % (matchobj.group(1), matchobj.group(1)) + + def process(matchobj): + data = matchobj.group(2) + data = RE_MD_INTERNAL_LINK.sub(makelink, data) + data = RE_MD_EXTERNAL_LINK.sub(link, data) + return matchobj.group(1) + data + + data = RE_MD_GET_INNER_HTML.sub(process, data) + return data + + +def render_markdown(data, auto_link=True): ''' returns the data as rendered markdown ''' - # cope with data == None if not data: return '' - return literal(ckan.misc.MarkdownFormat().to_html(data)) + data = RE_MD_HTML_TAGS.sub('', data.strip()) + data = markdown(data, safe_mode=True) + # tags can be added by tag:... or tag:"...." and a link will be made + # from it + if auto_link: + data = html_auto_link(data) + return literal(data) def format_resource_items(items): diff --git a/ckan/lib/mailer.py b/ckan/lib/mailer.py index c83b4d8fa45..f0e1978ac2c 100644 --- a/ckan/lib/mailer.py +++ b/ckan/lib/mailer.py @@ -7,12 +7,15 @@ from email import Utils from urlparse import urljoin -from pylons.i18n.translation import _ -from pylons import config, g -from ckan import model, __version__ -import ckan.lib.helpers as h +from pylons import config import paste.deploy.converters +import ckan +import ckan.model as model +import ckan.lib.helpers as h + +from ckan.common import _, g + log = logging.getLogger(__name__) class MailerException(Exception): @@ -36,7 +39,7 @@ def _mail_recipient(recipient_name, recipient_email, recipient = u"%s <%s>" % (recipient_name, recipient_email) msg['To'] = Header(recipient, 'utf-8') msg['Date'] = Utils.formatdate(time()) - msg['X-Mailer'] = "CKAN %s" % __version__ + msg['X-Mailer'] = "CKAN %s" % ckan.__version__ # Send the email using Python's smtplib. smtp_connection = smtplib.SMTP() diff --git a/ckan/lib/navl/dictization_functions.py b/ckan/lib/navl/dictization_functions.py index da0908e2dcc..aa4e69b59aa 100644 --- a/ckan/lib/navl/dictization_functions.py +++ b/ckan/lib/navl/dictization_functions.py @@ -1,9 +1,10 @@ import copy import formencode as fe import inspect -from pylons.i18n import _ from pylons import config +from ckan.common import _ + class Missing(object): def __unicode__(self): raise Invalid(_('Missing value')) diff --git a/ckan/lib/navl/validators.py b/ckan/lib/navl/validators.py index f72e2788617..01cbd87567f 100644 --- a/ckan/lib/navl/validators.py +++ b/ckan/lib/navl/validators.py @@ -1,5 +1,11 @@ -from dictization_functions import missing, StopOnError, Invalid -from pylons.i18n import _ +import ckan.lib.navl.dictization_functions as df + +from ckan.common import _ + +missing = df.missing +StopOnError = df.StopOnError +Invalid = df.Invalid + def identity_converter(key, data, errors, context): return diff --git a/ckan/lib/package_saver.py b/ckan/lib/package_saver.py index 1a1a9a8e7f6..5b3f876151f 100644 --- a/ckan/lib/package_saver.py +++ b/ckan/lib/package_saver.py @@ -1,4 +1,3 @@ -import genshi from sqlalchemy import orm import ckan.lib.helpers as h from ckan.lib.base import * @@ -22,12 +21,8 @@ def render_package(cls, pkg, context): render. Note that the actual calling of render('package/read') is left to the caller.''' - try: - notes_formatted = ckan.misc.MarkdownFormat().to_html(pkg.get('notes','')) - c.pkg_notes_formatted = genshi.HTML(notes_formatted) - except Exception, e: - error_msg = "%s" % _("Cannot render package description") - c.pkg_notes_formatted = genshi.HTML(error_msg) + c.pkg_notes_formatted = h.render_markdown(pkg.get('notes')) + c.current_rating, c.num_ratings = ckan.rating.get_rating(context['package']) url = pkg.get('url', '') c.pkg_url_link = h.link_to(url, url, rel='foaf:homepage', target='_blank') \ diff --git a/ckan/lib/search/query.py b/ckan/lib/search/query.py index 5d6c519ebac..21ab4261607 100644 --- a/ckan/lib/search/query.py +++ b/ckan/lib/search/query.py @@ -16,7 +16,7 @@ _open_licenses = None VALID_SOLR_PARAMETERS = set([ - 'q', 'fl', 'fq', 'rows', 'sort', 'start', 'wt', 'qf', 'bf', + 'q', 'fl', 'fq', 'rows', 'sort', 'start', 'wt', 'qf', 'bf', 'boost', 'facet', 'facet.mincount', 'facet.limit', 'facet.field', 'extras', 'fq_list', 'tie', 'defType', 'mm' ]) @@ -319,11 +319,6 @@ def run(self, query): rows_to_query = rows_to_return query['rows'] = rows_to_query - # order by score if no 'sort' term given - order_by = query.get('sort') - if order_by == 'rank' or order_by is None: - query['sort'] = 'score desc, name asc' - # show only results from this CKAN instance fq = query.get('fq', '') if not '+site_id:' in fq: @@ -358,6 +353,7 @@ def run(self, query): query['mm'] = query.get('mm', '2<-1 5<80%') query['qf'] = query.get('qf', QUERY_FIELDS) + conn = make_connection() log.debug('Package query: %r' % query) try: diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py index e21f222f50e..8f753c8ee5c 100644 --- a/ckan/logic/__init__.py +++ b/ckan/logic/__init__.py @@ -3,14 +3,12 @@ import types import re -from pylons.i18n import _ - -import ckan.lib.base as base import ckan.model as model -from ckan.new_authz import is_authorized -from ckan.lib.navl.dictization_functions import flatten_dict, DataError -from ckan.plugins import PluginImplementations -from ckan.plugins.interfaces import IActions +import ckan.new_authz as new_authz +import ckan.lib.navl.dictization_functions as df +import ckan.plugins as p + +from ckan.common import _, c log = logging.getLogger(__name__) @@ -174,7 +172,7 @@ def tuplize_dict(data_dict): try: key_list[num] = int(key) except ValueError: - raise DataError('Bad key') + raise df.DataError('Bad key') tuplized_dict[tuple(key_list)] = value return tuplized_dict @@ -190,7 +188,7 @@ def untuplize_dict(tuplized_dict): def flatten_to_string_key(dict): - flattented = flatten_dict(dict) + flattented = df.flatten_dict(dict) return untuplize_dict(flattented) @@ -205,7 +203,7 @@ def check_access(action, context, data_dict=None): # # TODO Check the API key is valid at some point too! # log.debug('Valid API key needed to make changes') # raise NotAuthorized - logic_authorization = is_authorized(action, context, data_dict) + logic_authorization = new_authz.is_authorized(action, context, data_dict) if not logic_authorization['success']: msg = logic_authorization.get('msg', '') raise NotAuthorized(msg) @@ -290,7 +288,7 @@ def get_action(action): # Then overwrite them with any specific ones in the plugins: resolved_action_plugins = {} fetched_actions = {} - for plugin in PluginImplementations(IActions): + for plugin in p.PluginImplementations(p.IActions): for name, auth_function in plugin.get_actions().items(): if name in resolved_action_plugins: raise Exception( @@ -317,7 +315,7 @@ def wrapped(context=None, data_dict=None, **kw): context.setdefault('model', model) context.setdefault('session', model.Session) try: - context.setdefault('user', base.c.user or base.c.author) + context.setdefault('user', c.user or c.author) except TypeError: # c not registered pass diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index f7d0913587f..feda4802cb1 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -1,9 +1,8 @@ import logging + from pylons import config -from pylons.i18n import _ from paste.deploy.converters import asbool -import ckan.new_authz as new_authz import ckan.lib.plugins as lib_plugins import ckan.logic as logic import ckan.rating as ratings @@ -15,6 +14,8 @@ import ckan.lib.dictization.model_save as model_save import ckan.lib.navl.dictization_functions +from ckan.common import _ + # FIXME this looks nasty and should be shared better from ckan.logic.action.update import _update_package_relationship diff --git a/ckan/logic/action/delete.py b/ckan/logic/action/delete.py index 8c4a7dd8cde..5dcaf88ab9b 100644 --- a/ckan/logic/action/delete.py +++ b/ckan/logic/action/delete.py @@ -1,9 +1,11 @@ -from pylons.i18n import _ import ckan.logic import ckan.logic.action import ckan.plugins as plugins import ckan.lib.dictization.model_dictize as model_dictize + +from ckan.common import _ + validate = ckan.lib.navl.dictization_functions.validate # Define some shortcuts diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index b9d9bda56c7..2cb0f9603bd 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -4,8 +4,6 @@ import datetime from pylons import config -from pylons.i18n import _ -from pylons import c import sqlalchemy import ckan.lib.dictization @@ -21,6 +19,8 @@ import ckan.lib.activity_streams as activity_streams import ckan.new_authz as new_authz +from ckan.common import _ + log = logging.getLogger('ckan.logic') # Define some shortcuts @@ -830,7 +830,7 @@ def _group_or_org_show(context, data_dict, is_org=False): _check_access('organization_show',context, data_dict) else: _check_access('group_show',context, data_dict) - + group_dict = model_dictize.group_dictize(group, context) @@ -1156,25 +1156,22 @@ def package_search(context, data_dict): This action accepts a *subset* of solr's search query parameters: + :param q: the solr query. Optional. Default: `"*:*"` :type q: string :param fq: any filter queries to apply. Note: `+site_id:{ckan_site_id}` is added to this string prior to the query being executed. :type fq: string - :param rows: the number of matching rows to return. - :type rows: int :param sort: sorting of the search results. Optional. Default: - "score desc, name asc". As per the solr documentation, this is a - comma-separated string of field names and sort-orderings. + 'relevance asc, metadata_modified desc'. As per the solr + documentation, this is a comma-separated string of field names and + sort-orderings. :type sort: string + :param rows: the number of matching rows to return. + :type rows: int :param start: the offset in the complete result for where the set of returned datasets should begin. :type start: int - :param qf: the dismax query fields to search within, including boosts. See - the `Solr Dismax Documentation - `_ - for further details. - :type qf: string :param facet: whether to enable faceted results. Default: "true". :type facet: string :param facet.mincount: the minimum counts for facet fields should be @@ -1187,6 +1184,18 @@ def package_search(context, data_dict): then the returned facet information is empty. :type facet.field: list of strings + + The following advanced Solr parameters are supported as well. Note that + some of these are only available on particular Solr versions. See Solr's + `dismax`_ and `edismax`_ documentation for further details on them: + + ``qf``, ``wt``, ``bf``, ``boost``, ``tie``, ``defType``, ``mm`` + + + .. _dismax: http://wiki.apache.org/solr/DisMaxQParserPlugin + .. _edismax: http://wiki.apache.org/solr/ExtendedDisMax + + **Results:** The result of this action is a dict with the following keys: @@ -1238,6 +1247,12 @@ def package_search(context, data_dict): _check_access('package_search', context, data_dict) + # Move ext_ params to extras and remove them from the root of the search + # params, so they don't cause and error + data_dict['extras'] = data_dict.get('extras', {}) + for key in [key for key in data_dict.keys() if key.startswith('ext_')]: + data_dict['extras'][key] = data_dict.pop(key) + # check if some extension needs to modify the search params for item in plugins.PluginImplementations(plugins.IPackageController): data_dict = item.before_search(data_dict) @@ -1246,6 +1261,9 @@ def package_search(context, data_dict): # the query abort = data_dict.get('abort_search',False) + if data_dict.get('sort') in (None, 'rank'): + data_dict['sort'] = 'score desc, metadata_created desc' + results = [] if not abort: # return a list of package ids @@ -1261,9 +1279,15 @@ def package_search(context, data_dict): if not 'capacity:' in p) data_dict['fq'] = fq + ' capacity:"public"' + # Pop these ones as Solr does not need them + extras = data_dict.pop('extras', None) + query = search.query_for(model.Package) query.run(data_dict) + # Add them back so extensions can use them on after_search + data_dict['extras'] = extras + for package in query.results: # get the package object package, package_dict = package['id'], package.get('data_dict') @@ -1301,7 +1325,8 @@ def package_search(context, data_dict): search_results = { 'count': count, 'facets': facets, - 'results': results + 'results': results, + 'sort': data_dict['sort'] } # Transform facets into a more useful data structure. @@ -2640,11 +2665,11 @@ def dashboard_activity_list_html(context, data_dict): ''' activity_stream = dashboard_activity_list(context, data_dict) + model = context['model'] offset = int(data_dict.get('offset', 0)) extra_vars = { - 'controller': 'dashboard', - 'action': 'activity', - 'id': data_dict['id'], + 'controller': 'user', + 'action': 'dashboard', 'offset': offset, } return activity_streams.activity_list_to_html(context, activity_stream, diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index d53a503ad41..d59365f7cc5 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -2,24 +2,24 @@ import datetime import json -import pylons -from pylons.i18n import _ from pylons import config from vdm.sqlalchemy.base import SQLAlchemySession -import paste.deploy.converters +import paste.deploy.converters as converters import ckan.plugins as plugins import ckan.logic as logic -import ckan.logic.schema -import ckan.lib.dictization +import ckan.logic.schema as schema_ +import ckan.lib.dictization as dictization import ckan.lib.dictization.model_dictize as model_dictize import ckan.lib.dictization.model_save as model_save import ckan.lib.navl.dictization_functions import ckan.lib.navl.validators as validators import ckan.lib.plugins as lib_plugins -import ckan.lib.email_notifications +import ckan.lib.email_notifications as email_notifications import ckan.lib.search as search +from ckan.common import _, request + log = logging.getLogger(__name__) # Define some shortcuts @@ -130,7 +130,7 @@ def related_update(context, data_dict): user = context['user'] id = _get_or_bust(data_dict, "id") - schema = context.get('schema') or ckan.logic.schema.default_related_schema() + schema = context.get('schema') or schema_.default_related_schema() related = model.Related.get(id) context["related"] = related @@ -170,7 +170,7 @@ def resource_update(context, data_dict): model = context['model'] user = context['user'] id = _get_or_bust(data_dict, "id") - schema = context.get('schema') or ckan.logic.schema.default_update_resource_schema() + schema = context.get('schema') or schema_.default_update_resource_schema() resource = model.Resource.get(id) context["resource"] = resource @@ -343,7 +343,7 @@ def package_relationship_update(context, data_dict): ''' model = context['model'] - schema = context.get('schema') or ckan.logic.schema.default_update_relationship_schema() + schema = context.get('schema') or schema_.default_update_relationship_schema() id, id2, rel = _get_or_bust(data_dict, ['subject', 'object', 'type']) @@ -424,7 +424,7 @@ def _group_or_org_update(context, data_dict, is_org=False): # when editing an org we do not want to update the packages if using the # new templates. if ((not is_org) - and not paste.deploy.converters.asbool( + and not converters.asbool( config.get('ckan.legacy_templates', False)) and 'api_version' not in context): context['prevent_packages_update'] = True @@ -481,7 +481,7 @@ def _group_or_org_update(context, data_dict, is_org=False): activity_dict['activity_type'] = 'deleted group' if activity_dict is not None: activity_dict['data'] = { - 'group': ckan.lib.dictization.table_dictize(group, context) + 'group': dictization.table_dictize(group, context) } activity_create_context = { 'model': model, @@ -552,7 +552,7 @@ def user_update(context, data_dict): model = context['model'] user = context['user'] session = context['session'] - schema = context.get('schema') or ckan.logic.schema.default_update_user_schema() + schema = context.get('schema') or schema_.default_update_user_schema() id = _get_or_bust(data_dict, 'id') user_obj = model.User.get(id) @@ -620,7 +620,7 @@ def task_status_update(context, data_dict): user = context['user'] id = data_dict.get("id") - schema = context.get('schema') or ckan.logic.schema.default_task_status_schema() + schema = context.get('schema') or schema_.default_task_status_schema() if id: task_status = model.TaskStatus.get(id) @@ -830,7 +830,7 @@ def vocabulary_update(context, data_dict): _check_access('vocabulary_update', context, data_dict) - schema = context.get('schema') or ckan.logic.schema.default_update_vocabulary_schema() + schema = context.get('schema') or schema_.default_update_vocabulary_schema() data, errors = _validate(data_dict, schema, context) if errors: model.Session.rollback() @@ -989,15 +989,15 @@ def send_email_notifications(context, data_dict): # If paste.command_request is True then this function has been called # by a `paster post ...` command not a real HTTP request, so skip the # authorization. - if not pylons.request.environ.get('paste.command_request'): + if not request.environ.get('paste.command_request'): _check_access('send_email_notifications', context, data_dict) - if not paste.deploy.converters.asbool( - pylons.config.get('ckan.activity_streams_email_notifications')): + if not converters.asbool( + config.get('ckan.activity_streams_email_notifications')): raise logic.ParameterError('ckan.activity_streams_email_notifications' ' is not enabled in config') - ckan.lib.email_notifications.get_and_send_notifications_for_all_users() + email_notifications.get_and_send_notifications_for_all_users() def package_owner_org_update(context, data_dict): diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index 64d5cc3e150..1baf9b61077 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -1,8 +1,8 @@ -from pylons.i18n import _ - import ckan.logic as logic import ckan.new_authz as new_authz +from ckan.common import _ + def package_create(context, data_dict=None): user = context['user'] diff --git a/ckan/logic/converters.py b/ckan/logic/converters.py index a1270a59146..bc6e9fdf5e4 100644 --- a/ckan/logic/converters.py +++ b/ckan/logic/converters.py @@ -1,9 +1,9 @@ -from pylons.i18n import _ -from ckan import model -from ckan.lib.navl.dictization_functions import Invalid -from ckan.lib.field_types import DateType, DateConvertError -from ckan.logic.validators import tag_length_validator, tag_name_validator, \ - tag_in_vocabulary_validator +import ckan.model as model +import ckan.lib.navl.dictization_functions as df +import ckan.lib.field_types as field_types +import ckan.logic.validators as validators + +from ckan.common import _ def convert_to_extras(key, data, errors, context): extras = data.get(('extras',), []) @@ -20,16 +20,16 @@ def convert_from_extras(key, data, errors, context): def date_to_db(value, context): try: - value = DateType.form_to_db(value) - except DateConvertError, e: - raise Invalid(str(e)) + value = field_types.DateType.form_to_db(value) + except field_types.DateConvertError, e: + raise df.Invalid(str(e)) return value def date_to_form(value, context): try: - value = DateType.db_to_form(value) - except DateConvertError, e: - raise Invalid(str(e)) + value = field_types.DateType.db_to_form(value) + except field_types.DateConvertError, e: + raise df.Invalid(str(e)) return value def free_tags_only(key, data, errors, context): @@ -56,11 +56,11 @@ def callable(key, data, errors, context): v = model.Vocabulary.get(vocab) if not v: - raise Invalid(_('Tag vocabulary "%s" does not exist') % vocab) + raise df.Invalid(_('Tag vocabulary "%s" does not exist') % vocab) context['vocabulary'] = v for tag in new_tags: - tag_in_vocabulary_validator(tag, context) + validators.tag_in_vocabulary_validator(tag, context) for num, tag in enumerate(new_tags): data[('tags', num + n, 'name')] = tag @@ -71,7 +71,7 @@ def convert_from_tags(vocab): def callable(key, data, errors, context): v = model.Vocabulary.get(vocab) if not v: - raise Invalid(_('Tag vocabulary "%s" does not exist') % vocab) + raise df.Invalid(_('Tag vocabulary "%s" does not exist') % vocab) tags = [] for k in data.keys(): @@ -103,7 +103,7 @@ def convert_user_name_or_id_to_id(user_name_or_id, context): result = session.query(model.User).filter_by( name=user_name_or_id).first() if not result: - raise Invalid('%s: %s' % (_('Not found'), _('User'))) + raise df.Invalid('%s: %s' % (_('Not found'), _('User'))) return result.id def convert_package_name_or_id_to_id(package_name_or_id, context): @@ -128,7 +128,7 @@ def convert_package_name_or_id_to_id(package_name_or_id, context): result = session.query(model.Package).filter_by( name=package_name_or_id).first() if not result: - raise Invalid('%s: %s' % (_('Not found'), _('Dataset'))) + raise df.Invalid('%s: %s' % (_('Not found'), _('Dataset'))) return result.id def convert_group_name_or_id_to_id(group_name_or_id, context): @@ -153,5 +153,5 @@ def convert_group_name_or_id_to_id(group_name_or_id, context): result = session.query(model.Group).filter_by( name=group_name_or_id).first() if not result: - raise Invalid('%s: %s' % (_('Not found'), _('Group'))) + raise df.Invalid('%s: %s' % (_('Not found'), _('Group'))) return result.id diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 07b22b6f9cb..515846284d5 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -116,6 +116,7 @@ def default_create_tag_schema(): def default_create_package_schema(): schema = { + '__before': [duplicate_extras_key, ignore], 'id': [empty], 'revision_id': [ignore], 'name': [not_empty, unicode, name_validator, package_name_validator], @@ -139,7 +140,6 @@ def default_create_package_schema(): 'tags': default_tags_schema(), 'tag_string': [ignore_missing, tag_string_convert], 'extras': default_extras_schema(), - 'extras_validation': [duplicate_extras_key, ignore], 'save': [ignore], 'return_to': [ignore], 'relationships_as_object': default_relationship_schema(), diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index b671231a41f..607494ade73 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -2,27 +2,32 @@ from itertools import count import re -from pylons.i18n import _ - -from ckan.lib.navl.dictization_functions import Invalid, StopOnError, Missing, missing, unflatten -from ckan.logic import check_access, NotAuthorized, NotFound +import ckan.lib.navl.dictization_functions as df +import ckan.logic as logic import ckan.lib.helpers as h from ckan.model import (MAX_TAG_LENGTH, MIN_TAG_LENGTH, PACKAGE_NAME_MIN_LENGTH, PACKAGE_NAME_MAX_LENGTH, PACKAGE_VERSION_MAX_LENGTH, VOCABULARY_NAME_MAX_LENGTH, VOCABULARY_NAME_MIN_LENGTH) -import ckan.new_authz +import ckan.new_authz as new_authz + +from ckan.common import _ + +Invalid = df.Invalid +StopOnError = df.StopOnError +Missing = df.Missing +missing = df.missing def owner_org_validator(key, data, errors, context): value = data.get(key) if value is missing or not value: - if not ckan.new_authz.check_config_permission('create_unowned_dataset'): + if not new_authz.check_config_permission('create_unowned_dataset'): raise Invalid(_('A organization must be supplied')) data.pop(key, None) - raise StopOnError + raise df.StopOnError model = context['model'] group = model.Group.get(value) @@ -303,7 +308,7 @@ def package_version_validator(value, context): def duplicate_extras_key(key, data, errors, context): - unflattened = unflatten(data) + unflattened = df.unflatten(data) extras = unflattened.get('extras', []) extras_keys = [] for extra in extras: @@ -313,7 +318,9 @@ def duplicate_extras_key(key, data, errors, context): for extra_key in set(extras_keys): extras_keys.remove(extra_key) if extras_keys: - errors[key].append(_('Duplicate key "%s"') % extras_keys[0]) + key_ = ('extras_validation',) + assert key_ not in errors + errors[key_] = [_('Duplicate key "%s"') % extras_keys[0]] def group_name_validator(key, data, errors, context): model = context['model'] @@ -392,16 +399,16 @@ def ignore_not_package_admin(key, data, errors, context): if 'ignore_auth' in context: return - if user and ckan.new_authz.is_sysadmin(user): + if user and new_authz.is_sysadmin(user): return authorized = False pkg = context.get('package') if pkg: try: - check_access('package_change_state',context) + logic.check_access('package_change_state',context) authorized = True - except NotAuthorized: + except logic.NotAuthorized: authorized = False if (user and pkg and authorized): @@ -419,16 +426,16 @@ def ignore_not_group_admin(key, data, errors, context): model = context['model'] user = context.get('user') - if user and ckan.new_authz.is_sysadmin(user): + if user and new_authz.is_sysadmin(user): return authorized = False group = context.get('group') if group: try: - check_access('group_change_state',context) + logic.check_access('group_change_state',context) authorized = True - except NotAuthorized: + except logic.NotAuthorized: authorized = False if (user and group and authorized): @@ -590,6 +597,6 @@ def user_name_exists(user_name, context): def role_exists(role, context): - if role not in ckan.new_authz.ROLE_PERMISSIONS: + if role not in new_authz.ROLE_PERMISSIONS: raise Invalid(_('role does not exist.')) return role diff --git a/ckan/misc.py b/ckan/misc.py deleted file mode 100644 index 29f3ae6447b..00000000000 --- a/ckan/misc.py +++ /dev/null @@ -1,95 +0,0 @@ -import re -import logging -import urllib -import webhelpers.markdown - -from pylons.i18n import _ - -log = logging.getLogger(__name__) - -class TextFormat(object): - - def to_html(self, instr): - raise NotImplementedError() - - -class MarkdownFormat(TextFormat): - internal_link = re.compile('(dataset|package|group):([a-z0-9\-_]+)') - - # tag names are allowed more characters, including spaces. So are - # treated specially. - internal_tag_link = re.compile(\ - r"""(tag): # group 1 - ( # capture name (inc. quotes) (group 2) - (")? # optional quotes for multi-word name (group 3) - ( # begin capture of the name w/o quotes (group 4) - (?(3) # if the quotes matched in group 3 - [ \w\-.] # then capture spaces (as well as other things) - | # else - [\w\-.] # don't capture spaces - ) # end - +) # end capture of the name w/o quotes (group 4) - (?(3)") # close opening quote if necessary - ) # end capture of the name with quotes (group 2) - """, re.VERBOSE|re.UNICODE) - normal_link = re.compile('<(http:[^>]+)>') - - html_whitelist = 'b center li ol p table td tr ul'.split(' ') - whitelist_elem = re.compile(r'<(\/?((%s)(\s[^>]*)?))>' % "|".join(html_whitelist), re.IGNORECASE) - whitelist_escp = re.compile(r'\\xfc\\xfd(\/?((%s)(\s[^>]*?)?))\\xfd\\xfc' % "|".join(html_whitelist), re.IGNORECASE) - normal_link = re.compile(r']*?href="([^"]*?)"[^>]*?>', re.IGNORECASE) - abbrev_link = re.compile(r'<(http://[^>]*)>', re.IGNORECASE) - any_link = re.compile(r']*?>', re.IGNORECASE) - close_link = re.compile(r'<(\/a[^>]*)>', re.IGNORECASE) - link_escp = re.compile(r'\\xfc\\xfd(\/?(%s)[^>]*?)\\xfd\\xfc' % "|".join(['a']), re.IGNORECASE) - web_address = re.compile(r'(\s|

)((http|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?)', re.IGNORECASE) - - def to_html(self, text): - if text is None: - return '' - # Encode whitelist elements. - text = self.whitelist_elem.sub(r'\\\\xfc\\\\xfd\1\\\\xfd\\\\xfc', text) - - # Encode links only in an acceptable format (guard against spammers) - text = self.normal_link.sub(r'\\\\xfc\\\\xfda href="\1" target="_blank" rel="nofollow"\\\\xfd\\\\xfc', text) - text = self.abbrev_link.sub(r'\\\\xfc\\\\xfda href="\1" target="_blank" rel="nofollow"\\\\xfd\\\\xfc\1', text) - text = self.any_link.sub(r'\\\\xfc\\\\xfda href="TAG MALFORMED" target="_blank" rel="nofollow"\\\\xfd\\\\xfc', text) - text = self.close_link.sub(r'\\\\xfc\\\\xfd\1\\\\xfd\\\\xfc', text) - - # Convert internal tag links - text = self.internal_tag_link.sub(self._create_tag_link, text) - - # Convert internal links. - text = self.internal_link.sub(r'[\1:\2] (/\1/\2)', text) - - # Convert to markdown format. - text = self.normal_link.sub(r'[\1] (\1)', text) - - # Convert to markdown format. - text = self.normal_link.sub(r'[\1] (\1)', text) - - # Markdown to HTML. - text = webhelpers.markdown.markdown(text, safe_mode=True) - - # Remaining unlinked web addresses to become addresses - text = self.web_address.sub(r'\1\2', text) - - # Decode whitelist elements. - text = self.whitelist_escp.sub(r'<\1>', text) - text = self.link_escp.sub(r'<\1>', text) - - return text - - def _create_tag_link(self, match_object): - """ - A callback used to create the internal tag link. - - The reason for this is that webhelpers.markdown does not percent-escape - spaces, nor does it encode unicode characters correctly. - - This is only applied to the tag substitution since only tags may - have spaces or unicode characters. - """ - g = match_object.group - url = urllib.quote(g(4).encode('utf8')) - return r'[%s:%s] (/%s/%s)' % (g(1), g(2), g(1), url) diff --git a/ckan/model/__init__.py b/ckan/model/__init__.py index 7b3d7e5887a..96812963a17 100644 --- a/ckan/model/__init__.py +++ b/ckan/model/__init__.py @@ -101,6 +101,7 @@ from tracking import ( tracking_summary_table, TrackingSummary, + tracking_raw_table ) from rating import ( Rating, diff --git a/ckan/model/license.py b/ckan/model/license.py index 2a0b8e732e8..1b65d6040ec 100644 --- a/ckan/model/license.py +++ b/ckan/model/license.py @@ -1,10 +1,10 @@ import datetime import urllib2 import re -import simplejson as json from pylons import config -from pylons.i18n import _ + +from ckan.common import _, json class License(object): diff --git a/ckan/model/package.py b/ckan/model/package.py index ec49bbab019..bce320bde98 100644 --- a/ckan/model/package.py +++ b/ckan/model/package.py @@ -17,7 +17,6 @@ import activity import extension -import ckan.misc import ckan.lib.dictization __all__ = ['Package', 'package_table', 'package_revision_table', @@ -216,7 +215,8 @@ def as_dict(self, ref_package_by='name', ref_group_by='name'): if self.metadata_modified else None _dict['metadata_created'] = self.metadata_created.isoformat() \ if self.metadata_created else None - _dict['notes_rendered'] = ckan.misc.MarkdownFormat().to_html(self.notes) + import ckan.lib.helpers as h + _dict['notes_rendered'] = h.render_markdown(self.notes) _dict['type'] = self.type or u'dataset' #tracking import ckan.model as model diff --git a/ckan/model/package_relationship.py b/ckan/model/package_relationship.py index 503fc45b654..3ee5cd49a7f 100644 --- a/ckan/model/package_relationship.py +++ b/ckan/model/package_relationship.py @@ -10,7 +10,7 @@ # i18n only works when this is run as part of pylons, # which isn't the case for paster commands. try: - from pylons.i18n import _ + from ckan.common import _ _('') except: def _(txt): diff --git a/ckan/model/tracking.py b/ckan/model/tracking.py index c10684960ea..dca69212a83 100644 --- a/ckan/model/tracking.py +++ b/ckan/model/tracking.py @@ -3,7 +3,15 @@ import meta import domain_object -__all__ = ['tracking_summary_table', 'TrackingSummary'] +__all__ = ['tracking_summary_table', 'TrackingSummary', 'tracking_raw_table'] + +tracking_raw_table = Table('tracking_raw', meta.metadata, + Column('user_key', types.Unicode(100), nullable=False), + Column('url', types.UnicodeText, nullable=False), + Column('tracking_type', types.Unicode(10), nullable=False), + Column('access_timestamp', types.DateTime), + ) + tracking_summary_table = Table('tracking_summary', meta.metadata, Column('url', types.UnicodeText, primary_key=True, nullable=False), diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py index e011184fe56..25882e9395a 100644 --- a/ckan/plugins/toolkit.py +++ b/ckan/plugins/toolkit.py @@ -2,7 +2,6 @@ import os import re -import pylons import paste.deploy.converters as converters import webhelpers.html.tags @@ -74,6 +73,7 @@ def _initialize(self): import ckan.logic as logic import ckan.lib.cli as cli import ckan.lib.plugins as lib_plugins + import ckan.common as common # Allow class access to these modules self.__class__.ckan = ckan @@ -82,9 +82,9 @@ def _initialize(self): t = self._toolkit # imported functions - t['_'] = pylons.i18n._ - t['c'] = pylons.c - t['request'] = pylons.request + t['_'] = common._ + t['c'] = common.c + t['request'] = common.request t['render'] = base.render t['render_text'] = base.render_text t['asbool'] = converters.asbool diff --git a/ckan/public/base/css/main.css b/ckan/public/base/css/main.css index b4b5fc9cb69..0ea0e13e359 100644 --- a/ckan/public/base/css/main.css +++ b/ckan/public/base/css/main.css @@ -6513,6 +6513,11 @@ textarea { -moz-border-radius-bottomright: 2px; border-bottom-right-radius: 2px; } +.ellipsis { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} .ckan-icon { *margin-right: .3em; display: inline-block; diff --git a/ckan/public/base/less/prose.less b/ckan/public/base/less/prose.less index c3846d7db27..315668a0227 100644 --- a/ckan/public/base/less/prose.less +++ b/ckan/public/base/less/prose.less @@ -68,3 +68,9 @@ border-bottom-right-radius: @radius; } } + +.ellipsis { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} diff --git a/ckan/templates/activity_streams/activity_stream_items.html b/ckan/templates/activity_streams/activity_stream_items.html index ff863425a5a..42e54acd2b4 100644 --- a/ckan/templates/activity_streams/activity_stream_items.html +++ b/ckan/templates/activity_streams/activity_stream_items.html @@ -4,7 +4,7 @@ {% if activities %}

    {% if offset > 0 %} -
  • {{ _('Load less') }}
    • +
  • {{ _('Load less') }}
    • {% endif %} {% for activity in activities %} {% if loop.index <= has_more_length %} diff --git a/ckan/templates/organization/index.html b/ckan/templates/organization/index.html index 1ac66970154..74ef2cd9050 100644 --- a/ckan/templates/organization/index.html +++ b/ckan/templates/organization/index.html @@ -1,6 +1,6 @@ {% extends "page.html" %} -{% block subtitle %}{{ _('Organizations of Datasets') }}{% endblock %} +{% block subtitle %}{{ _('Organizations') }}{% endblock %} {% block breadcrumb_content %}
  • {% link_for _('Organizations'), controller='organization', action='index' %}
    • diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index 80156ff93dd..6e607ea3407 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -46,7 +46,7 @@ {% block resource_read_title %}

    {{ h.resource_display_name(res) | truncate(50) }}

    {% endblock %} {% block resource_read_url %} {% if res.url %} -

    {{ _('URL:') }} {{ res.url }}

    +

    {{ _('URL:') }} {{ res.url }}

    {% endif %} {% endblock %}
    diff --git a/ckan/templates/snippets/sort_by.html b/ckan/templates/snippets/sort_by.html index a2a6076c3bc..c568e2464f9 100644 --- a/ckan/templates/snippets/sort_by.html +++ b/ckan/templates/snippets/sort_by.html @@ -11,11 +11,13 @@ diff --git a/ckan/templates_legacy/package/read_core.html b/ckan/templates_legacy/package/read_core.html index 9868845a8ec..427775a8d42 100644 --- a/ckan/templates_legacy/package/read_core.html +++ b/ckan/templates_legacy/package/read_core.html @@ -10,7 +10,7 @@
    -
    +
    ${c.pkg_notes_formatted}
    diff --git a/ckan/tests/functional/api/test_dashboard.py b/ckan/tests/functional/api/test_dashboard.py index eb7c805702a..e793244ebaf 100644 --- a/ckan/tests/functional/api/test_dashboard.py +++ b/ckan/tests/functional/api/test_dashboard.py @@ -331,3 +331,20 @@ def test_09_activities_that_should_not_show(self): after = self.dashboard_activity_list(self.new_user) assert before == after + + def test_10_dashboard_activity_list_html_does_not_crash(self): + + params = json.dumps({'name': 'irrelevant_dataset1'}) + response = self.app.post('/api/action/package_create', params=params, + extra_environ={'Authorization': str(self.annafan['apikey'])}) + assert response.json['success'] is True + + params = json.dumps({'name': 'another_irrelevant_dataset'}) + response = self.app.post('/api/action/package_create', params=params, + extra_environ={'Authorization': str(self.annafan['apikey'])}) + assert response.json['success'] is True + + res = self.app.get('/api/3/action/dashboard_activity_list_html', + extra_environ={'Authorization': + str(self.annafan['apikey'])}) + assert res.json['success'] is True diff --git a/ckan/tests/functional/test_home.py b/ckan/tests/functional/test_home.py index f69ca90d5da..248810b56ab 100644 --- a/ckan/tests/functional/test_home.py +++ b/ckan/tests/functional/test_home.py @@ -1,4 +1,3 @@ -from pylons import c, session from pylons.i18n import set_lang from ckan.lib.create_test_data import CreateTestData @@ -10,6 +9,8 @@ from ckan.tests.pylons_controller import PylonsTestCase from ckan.tests import search_related, setup_test_search_index +from ckan.common import c, session + class TestHomeController(TestController, PylonsTestCase, HtmlCheckMethods): @classmethod def setup_class(cls): diff --git a/ckan/tests/functional/test_package.py b/ckan/tests/functional/test_package.py index 4adabcc6c78..4c2adeb3de4 100644 --- a/ckan/tests/functional/test_package.py +++ b/ckan/tests/functional/test_package.py @@ -309,10 +309,6 @@ def test_read(self): assert anna.version in res assert anna.url in res assert 'Some test notes' in res - self.check_named_element(res, 'a', - 'http://ckan.net/', - 'target="_blank"', - 'rel="nofollow"') assert 'Some bolded text.' in res self.check_tag_and_data(res, 'left arrow', '<') self.check_tag_and_data(res, 'umlaut', u'\xfc') @@ -350,7 +346,7 @@ def test_read_internal_links(self): pkg_name = u'link-test', CreateTestData.create_arbitrary([ {'name':pkg_name, - 'notes':'Decoy link here: decoy:decoy, real links here: package:pkg-1, ' \ + 'notes':'Decoy link here: decoy:decoy, real links here: dataset:pkg-1, ' \ 'tag:tag_1 group:test-group-1 and a multi-word tag: tag:"multi word with punctuation."', } ]) @@ -358,9 +354,9 @@ def test_read_internal_links(self): res = self.app.get(offset) def check_link(res, controller, id): id_in_uri = id.strip('"').replace(' ', '%20') # remove quotes and percent-encode spaces - self.check_tag_and_data(res, 'a ', '/%s/%s' % (controller, id_in_uri), - '%s:%s' % (controller, id)) - check_link(res, 'package', 'pkg-1') + self.check_tag_and_data(res, 'a ', '%s/%s' % (controller, id_in_uri), + '%s:%s' % (controller, id.replace('"', '"'))) + check_link(res, 'dataset', 'pkg-1') check_link(res, 'tag', 'tag_1') check_link(res, 'tag', '"multi word with punctuation."') check_link(res, 'group', 'test-group-1') @@ -1557,10 +1553,10 @@ def teardown(self): def test_markdown_html_whitelist(self): self.body = str(self.res) - self.assert_fragment('') - self.assert_fragment('') - self.assert_fragment('subcategory.txt') - self.assert_fragment('') + self.fail_if_fragment('
    Description
    --
    ') + self.fail_if_fragment('') + self.fail_if_fragment('subcategory.txt') + self.fail_if_fragment('') self.fail_if_fragment('
    Description
    --