From 845c0e5769edb6a1e66a86d97e3ae67f10841363 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 31 May 2012 11:13:14 +0100 Subject: [PATCH 001/155] [#2389] Move recline js and css to a snippet This will make easier to modify the libraries and load them conditionally. --- ckan/templates/package/resource_read.html | 36 ++---------------- .../snippets/recline-extra-footer.html | 13 +++++++ .../snippets/recline-extra-header.html | 38 +++++++++++++++++++ 3 files changed, 54 insertions(+), 33 deletions(-) create mode 100644 ckan/templates/snippets/recline-extra-footer.html create mode 100644 ckan/templates/snippets/recline-extra-header.html diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index bebe246a0ae..cf03b67f502 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -15,36 +15,9 @@ - - - - - - - - - + + From d1c82313143b77249223d6da56823c51d1255137 Mon Sep 17 00:00:00 2001 From: "Sven R. Kunze" Date: Mon, 18 Jun 2012 23:34:15 +0200 Subject: [PATCH 002/155] Added route 'user/dashboard' and action 'dashboard' to user controller. --- ckan/config/routing.py | 1 + ckan/controllers/user.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/ckan/config/routing.py b/ckan/config/routing.py index 3890032817f..b529d90eed7 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -252,6 +252,7 @@ def make_map(): m.connect('/user/edit', action='edit') # Note: openid users have slashes in their ids, so need the wildcard # in the route. + m.connect('/user/dashboard', action='dashboard') m.connect('/user/followers/{id:.*}', action='followers') m.connect('/user/edit/{id:.*}', action='edit') m.connect('/user/reset/{id:.*}', action='perform_reset') diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index c1b81dc3070..ebcfeaa9421 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -444,3 +444,16 @@ def followers(self, id=None): c.followers = get_action('user_follower_list')(context, {'id':c.user_dict['id']}) return render('user/followers.html') + + + def dashboard(self, id=None): + context = {'model': model, 'session': model.Session, + 'user': c.user or c.author, 'for_view': True} + data_dict = {'id':id, 'user_obj':c.userobj} + self._setup_template_variables(context, data_dict) + c.followers = get_action('user_follower_list')(context, + {'id':c.user_dict['id']}) + return render('user/followers.html') + + + From b26daa5cc74f9914de317012edd58d1d17283147 Mon Sep 17 00:00:00 2001 From: "Sven R. Kunze" Date: Tue, 19 Jun 2012 15:04:08 +0200 Subject: [PATCH 003/155] New template 'dashboard.html', modified layout --- ckan/controllers/user.py | 4 +--- ckan/templates/user/dashboard.html | 16 ++++++++++++++++ ckan/templates/user/layout.html | 1 + 3 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 ckan/templates/user/dashboard.html diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index ebcfeaa9421..ed967b781e7 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -451,9 +451,7 @@ def dashboard(self, id=None): 'user': c.user or c.author, 'for_view': True} data_dict = {'id':id, 'user_obj':c.userobj} self._setup_template_variables(context, data_dict) - c.followers = get_action('user_follower_list')(context, - {'id':c.user_dict['id']}) - return render('user/followers.html') + return render('user/dashboard.html') diff --git a/ckan/templates/user/dashboard.html b/ckan/templates/user/dashboard.html new file mode 100644 index 00000000000..139443aefca --- /dev/null +++ b/ckan/templates/user/dashboard.html @@ -0,0 +1,16 @@ + + + ${c.user} - Dashboard - User + + ${c.user_dict.display_name}'s Dashboard + + +
+ Hello World. +
+ + + diff --git a/ckan/templates/user/layout.html b/ckan/templates/user/layout.html index a61778d9d39..f0fbd5dcafd 100644 --- a/ckan/templates/user/layout.html +++ b/ckan/templates/user/layout.html @@ -8,6 +8,7 @@
    +
  • Dashboard
  • My Profile
  • Edit Profile
  • Log out
    • From 958d00408712c4a3c6b8bd535323ef7c140288de Mon Sep 17 00:00:00 2001 From: "Sven R. Kunze" Date: Wed, 20 Jun 2012 12:52:03 +0200 Subject: [PATCH 004/155] UserFollowingUser and UserFollowingDataset classes extended to discover followed users and datasets; inlcuded in user controller and template; contains bug: no datasets are displayed --- ckan/controllers/user.py | 4 +- ckan/logic/action/get.py | 55 +++++++++++++++++++++- ckan/model/follower.py | 74 +++++++++++++++++++++--------- ckan/templates/user/dashboard.html | 18 +++++++- 4 files changed, 126 insertions(+), 25 deletions(-) diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index ed967b781e7..e2a0060bf99 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -445,12 +445,14 @@ def followers(self, id=None): {'id':c.user_dict['id']}) return render('user/followers.html') - def dashboard(self, id=None): context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True} data_dict = {'id':id, 'user_obj':c.userobj} self._setup_template_variables(context, data_dict) + c.followed_users = get_action('followed_user_list')(context, {'id':c.user_dict['id']}) + c.followed_datasets = get_action('followed_dataset_list')(context, {'id':c.user_dict['id']}) + return render('user/dashboard.html') diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 191a72f7fea..8a98950b0d1 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1869,7 +1869,7 @@ def _follower_list(context, data_dict, FollowerClass): users = [model.User.get(follower.follower_id) for follower in followers] users = [user for user in users if user is not None] - # Dictize the list of user objects. + # Dictize the list of User objects. return [model_dictize.user_dictize(user,context) for user in users] def user_follower_list(context, data_dict): @@ -1951,3 +1951,56 @@ def am_following_dataset(context, data_dict): return _am_following(context, data_dict, context['model'].UserFollowingDataset) + + +def followed_user_list(context, data_dict): + '''Return the list of users that are followed by the given user. + + :param id: the id or name of the user + :type id: string + :rtype: list of dictionaries + + ''' + schema = context.get('schema') or ( + ckan.logic.schema.default_follow_user_schema()) + data_dict, errors = _validate(data_dict, schema, context) + if errors: + raise ValidationError(errors, ckan.logic.action.error_summary(errors)) + + # Get the list of Follower objects. + model = context['model'] + user_id = data_dict.get('id') + followeds = model.UserFollowingUser.followed_list(user_id) + + # Convert the list of Follower objects to a list of User objects. + users = [model.User.get(followed.object_id) for followed in followeds] + users = [user for user in users if user is not None] + + # Dictize the list of User objects. + return [model_dictize.user_dictize(user, context) for user in users] + +def followed_dataset_list(context, data_dict): + '''Return the list of datasets that are followed by the given user. + + :param id: the id or name of the user + :type id: string + :rtype: list of dictionaries + + ''' + schema = context.get('schema') or ( + ckan.logic.schema.default_follow_user_schema()) + data_dict, errors = _validate(data_dict, schema, context) + if errors: + raise ValidationError(errors, ckan.logic.action.error_summary(errors)) + + # Get the list of Follower objects. + model = context['model'] + user_id = data_dict.get('id') + followeds = model.UserFollowingDataset.followed_list(user_id) + + # Convert the list of Follower objects to a list of Package objects. + packages = [model.Package.get(followed.object_id) for followed in followeds] + packages = [package for package in packages if package is not None] + + # Dictize the list of Package objects. + return [model_dictize.package_dictize(package, context) for package in packages] diff --git a/ckan/model/follower.py b/ckan/model/follower.py index 0698867682c..b270a964d2e 100644 --- a/ckan/model/follower.py +++ b/ckan/model/follower.py @@ -27,24 +27,39 @@ def get(self, follower_id, object_id): return query.first() @classmethod - def follower_count(cls, object_id): - '''Return the number of users following a user.''' + def is_following(cls, follower_id, object_id): + '''Return True if follower_id is currently following object_id, False + otherwise. + + ''' + return UserFollowingUser.get(follower_id, object_id) is not None + + + @classmethod + def followed_count(cls, follower_id): + '''Return the number of the users being followed by a user.''' return meta.Session.query(UserFollowingUser).filter( - UserFollowingUser.object_id == object_id).count() + UserFollowingUser.follower_id == follower_id).count() @classmethod - def follower_list(cls, object_id): - '''Return a list of all of the followers of a user.''' + def followed_list(cls, follower_id): + '''Return a list of the users being followed by a user.''' return meta.Session.query(UserFollowingUser).filter( - UserFollowingUser.object_id == object_id).all() + UserFollowingUser.follower_id == follower_id).all() + @classmethod - def is_following(cls, follower_id, object_id): - '''Return True if follower_id is currently following object_id, False - otherwise. + def follower_count(cls, user_id): + '''Return the number of the followers of a user.''' + return meta.Session.query(UserFollowingUser).filter( + UserFollowingUser.object_id == user_id).count() + + @classmethod + def follower_list(cls, user_id): + '''Return a list of the followers of a user.''' + return meta.Session.query(UserFollowingUser).filter( + UserFollowingUser.object_id == user_id).all() - ''' - return UserFollowingUser.get(follower_id, object_id) is not None user_following_user_table = sqlalchemy.Table('user_following_user', meta.metadata, @@ -85,24 +100,39 @@ def get(self, follower_id, object_id): return query.first() @classmethod - def follower_count(cls, object_id): - '''Return the number of users following a dataset.''' + def is_following(cls, follower_id, object_id): + '''Return True if follower_id is currently following object_id, False + otherwise. + + ''' + return UserFollowingDataset.get(follower_id, object_id) is not None + + + @classmethod + def followed_count(cls, follower_id): + '''Return the number of the datasets being followed by a user.''' return meta.Session.query(UserFollowingDataset).filter( - UserFollowingDataset.object_id == object_id).count() + UserFollowingDataset.follower_id == follower_id).count() @classmethod - def follower_list(cls, object_id): - '''Return a list of all of the followers of a dataset.''' + def followed_list(cls, follower_id): + '''Return a list of the datasets being followed by a user.''' return meta.Session.query(UserFollowingDataset).filter( - UserFollowingDataset.object_id == object_id).all() + UserFollowingDataset.follower_id == follower_id).all() + @classmethod - def is_following(cls, follower_id, object_id): - '''Return True if follower_id is currently following object_id, False - otherwise. + def follower_count(cls, dataset_id): + '''Return the number of the followers of a dataset.''' + return meta.Session.query(UserFollowingDataset).filter( + UserFollowingDataset.object_id == dataset_id).count() + + @classmethod + def follower_list(cls, dataset_id): + '''Return a list of the followers of a dataset.''' + return meta.Session.query(UserFollowingDataset).filter( + UserFollowingDataset.object_id == dataset_id).all() - ''' - return UserFollowingDataset.get(follower_id, object_id) is not None user_following_dataset_table = sqlalchemy.Table('user_following_dataset', meta.metadata, diff --git a/ckan/templates/user/dashboard.html b/ckan/templates/user/dashboard.html index 139443aefca..20681c9577f 100644 --- a/ckan/templates/user/dashboard.html +++ b/ckan/templates/user/dashboard.html @@ -8,8 +8,24 @@ ${c.user_dict.display_name}'s Dashboard +

    XXXX ${len(c.followed_users)} XXXX

    +

    XXXX ${len(c.followed_datasets)} XXXX

    +
    - Hello World. +
      +
    • +
        + ${h.linked_user(followed_user['name'], maxlength=20)} +
      +
      • +
    +
      +
    • +
        + ${h.dataset_link(followed_dataset['name'])} +
      +
      • +
    From bc0caafcc6b83aa1fd6a7a5c3d078d435b8f0372 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Wed, 20 Jun 2012 12:14:08 +0100 Subject: [PATCH 005/155] [#2437] Add standard python coding standards. --- doc/coding-standards.rst | 117 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 115 insertions(+), 2 deletions(-) diff --git a/doc/coding-standards.rst b/doc/coding-standards.rst index 770c6a8a631..15bcfd0e6d1 100644 --- a/doc/coding-standards.rst +++ b/doc/coding-standards.rst @@ -74,15 +74,109 @@ Frontend Coding Standards ========================= TODO +- i18n + + - In html, don't include line breaks within ``

    `` blocks. ie do this: :: + +

    Blah foo blah

    +

    New paragraph, blah

    + + And **not**: :: + +

    Blah foo blah + New paragraph, blah

    http://aron.github.com/ckan-style/styleguide/ Backend Coding Standards ======================== -TODO +For python code, we follow `PEP 8`_, plus a few of our own rules. The +important bits are laid out below, but if in doubt, refer to `PEP 8`_ and +common sense. + +Layout and formatting +--------------------- + +- Don't use tabs. Use 4 spaces. + +- Maximum line length is 79 characters. + +- Continuation lines should align vertically within the parentheses, or with + a hanging indent. See `PEP 8's Indent Section`_ for more details. + +- Avoid extraneous whitespace. See `PEP 8's Whitespace Section`_ for more details. + +- Clean up formatting issues in master, not on a feature branch. Unless of + course you're changing that piece of code anyway. This will help avoid + spurious merge conflicts, and aid in reading pull requests. + +- Use the single-quote character, ``'``, rather than the double-quote + character, ``"``, for string literals. + +.. _PEP 8: http://www.python.org/dev/peps/pep-0008/ +.. _PEP 8's Indent Section: http://www.python.org/dev/peps/pep-0008/#indentation +.. _PEP 8's Whitespace Section: http://www.python.org/dev/peps/pep-0008/#whitespace-in-expressions-and-statements + +Imports +------- + +- Import whole modules, rather than using ``from foo import bar``. It's ok + to alias imported modules to make things more concise, ie this *is* + acceptable: :: + + import foo.bar.baz as f + +- Make all imports at the start of the file, after the module docstring. + Imports should be grouped in the following order: -http://wiki.okfn.org/Coding_Standards + 1. Standard library imports + 2. Third-party imports + 3. CKAN imports + +Logging +------- + +- Keep messages short. + +- Don't include object representations in the log message. It **is** useful + to include an domain model identifier where appropriate. + +- Choose an appropriate log-level: + + +----------+--------------------------------------------------------------+ + | Level | Description | + +==========+==============================================================+ + | DEBUG | Detailed information, of no interest when everything is | + | | working well but invaluable when diagnosing problems. | + +----------+--------------------------------------------------------------+ + | INFO | Affirmations that things are working as expected, e.g. | + | | "service has started" or "indexing run complete". Often | + | | ignored. | + +----------+--------------------------------------------------------------+ + | WARNING | There may be a problem in the near future, and this gives | + | | advance warning of it. But the application is able to proceed| + | | normally. | + +----------+--------------------------------------------------------------+ + | ERROR | The application has been unable to proceed as expected, due | + | | to the problem being logged. | + +----------+--------------------------------------------------------------+ + | CRITICAL | This is a serious error, and some kind of application | + | | meltdown might be imminent. | + +----------+--------------------------------------------------------------+ + + (`Source + `_) + +i18n +---- + +To construct an internationalised string, use `str.format`_, giving +meaningful names to each replacement field. For example: :: + + _(' ... {foo} ... {bar} ...').format(foo='foo-value', bar='bar-value') + +.. _str.format: http://docs.python.org/library/stdtypes.html#str.format Docstring Standards ------------------- @@ -222,3 +316,22 @@ Example of a ckan.logic.action API docstring: ''' .. _Autodoc: http://sphinx.pocoo.org/ext/autodoc.html + +Tools +----- + +Running the `PEP 8 style guide checker`_ is good for checking adherence to `PEP +8`_ formatting. As mentioned above, only perform style clean-ups on master to +help avoid spurious merge conflicts. + +`PyLint`_ is a useful tool for analysing python source code for errors and signs of poor quality. + +`pyflakes`_ is another useful tool for passive analysis of python source code. +There's also a `pyflakes vim plugin`_ which will highlight unused variables, +undeclared variables, syntax errors and unused imports. + +.. _PEP 8 style guide checker: http://pypi.python.org/pypi/pep8 +.. _PyLint: http://www.logilab.org/857 +.. _pyflakes: http://pypi.python.org/pypi/pyflakes +.. _pyflakes vim plugin: http://www.vim.org/scripts/script.php?script_id=2441 + From c8147bed3fb4dfddf969d2568050c53de1308361 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Wed, 20 Jun 2012 12:58:25 +0100 Subject: [PATCH 006/155] [#2437] Import aron's javascript style guide. From http://aron.github.com/ckan-style/styleguide/javascript/ --- doc/coding-standards.rst | 376 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 375 insertions(+), 1 deletion(-) diff --git a/doc/coding-standards.rst b/doc/coding-standards.rst index 15bcfd0e6d1..76948250f8d 100644 --- a/doc/coding-standards.rst +++ b/doc/coding-standards.rst @@ -73,7 +73,381 @@ When merging a feature or bug branch into master: Frontend Coding Standards ========================= -TODO +Javavscript +----------- + +Formatting +`````````` + +.. _OKFN Coding Standards: http://wiki.okfn.org/Coding_Standards#Javascript +.. _idiomatic.js: https://github.com/rwldrn/idiomatic.js/ +.. _Douglas Crockford's: http://javascript.crockford.com/code.html + +All JavaScript documents must use **two spaces** for indentation and files +should have no trailing whitespace. This is contrary to the `OKFN Coding +Standards`_ but matches what's in use in the current code base. + +Coding style must follow the `idiomatic.js`_ style but with the following +exceptions. + +.. note:: Idiomatic is heavily based upon `Douglas Crockford's`_ style + guide which is recommended by the `OKFN Coding Standards`_. + +White Space +........... + +Two spaces must be used for indentation at all times. Unlike in idiomatic +whitespace must not be used _inside_ parentheses between the parentheses +and their Contents. :: + + // BAD: Too much whitespace. + function getUrl( full ) { + var url = '/styleguide/javascript/'; + if ( full ) { + url = 'http://okfn.github.com/ckan' + url; + } + return url; + } + + // GOOD: + function getUrl(full) { + var url = '/styleguide/javascript/'; + if (full) { + url = 'http://okfn.github.com/ckan' + url; + } + return url; + } + +.. note:: See section 2.D.1.1 of idiomatic for more examples of this syntax. + +Quotes +...... + +Single quotes should be used everywhere unless writing JSON or the string +contains them. This makes it easier to create strings containing HTML. :: + + jQuery('
    ').appendTo('body'); + +Object properties need not be quoted unless required by the interpreter. :: + + var object = { + name: 'bill', + 'class': 'user-name' + }; + +Variable declarations +..................... + +One ``var`` statement must be used per variable assignment. These must be +declared at the top of the function in which they are being used. :: + + // GOOD: + var good = "string"; + var alsoGood = "another; + + // GOOD: + var good = "string"; + var okay = [ + "hmm", "a bit", "better" + ]; + + // BAD: + var good = "string", + iffy = [ + "hmm", "not", "great" + ]; + +Declare variables at the top of the function in which they are first used. This +avoids issues with variable hoisting. If a variable is not assigned a value +until later in the function then it it okay to define more than one per +statement. :: + + // BAD: contrived example. + function lowercaseNames(names) { + var names = []; + + for (var index = 0, length = names.length; index < length; index += 1) { + var name = names[index]; + names.push(name.toLowerCase()); + } + + var sorted = names.sort(); + return sorted; + } + + // GOOD: + function lowercaseNames(names) { + var names = []; + var index, sorted, name; + + for (index = 0, length = names.length; index < length; index += 1) { + name = names[index]; + names.push(names[index].toLowerCase()); + } + + sorted = names.sort(); + return sorted; + } + +Naming +`````` + +All properties, functions and methods must use lowercase camelCase: :: + + var myUsername = 'bill'; + var methods = { + getSomething: function () {} + }; + +Constructor functions must use uppercase CamelCase: :: + + function DatasetSearchView() { + } + +Constants must be uppercase with spaces delimited by underscores: :: + + var env = { + PRODUCTION: 'production', + DEVELOPMENT: 'development', + TESTING: 'testing' + }; + +Event handlers and callback functions should be prefixed with "on": :: + + function onDownloadClick(event) {} + + jQuery('.download').click(onDownloadClick); + +Boolean variables or methods returning boolean functions should prefix +the variable name with "is": :: + + function isAdmin() {} + + var canEdit = isUser() && isAdmin(); + + +.. note:: Alternatives are "has", "can" and "should" if they make more sense + +Private methods should be prefixed with an underscore: :: + + View.extend({ + "click": "_onClick", + _onClick: function (event) { + } + }); + +Functions should be declared as named functions rather than assigning an +anonymous function to a variable. :: + + // GOOD: + function getName() { + } + + // BAD: + var getName = function () { + }; + +Named functions are generally easier to debug as they appear named in the +debugger. + +Comments +```````` + +Comments should be used to explain anything that may be unclear when you return +to it in six months time. Single line comments should be used for all inline +comments that do not form part of the documentation. :: + + // Export the function to either the exports or global object depending + // on the current environment. This can be either an AMD module, CommonJS + // module or a browser. + if (typeof module.define === 'function' && module.define.amd) { + module.define('broadcast', function () { + return Broadcast; + }); + } else if (module.exports) { + module.exports = Broadcast; + } else { + module.Broadcast = Broadcast; + } + +File Structure +`````````````` + +All public JavaScript files should be contained within a _javascript_ directory +within the _public_ directory and files should be structured accordingly. :: + + lib/ + main.js + utils.js + components/ + vendor/ + jquery.js + jquery.plugin.js + underscore.js + templates/ + test/ + index.html + spec/ + main-spec.js + utils-spec.js + vendor/ + mocha.js + mocha.css + chai.js + +All files and directories should be lowercase with hyphens used to separate words. + +lib + Should contain all application files. These can be structured appropriately. + It is recommended that *main.js* be used as the bootstrap filename that sets + up the page. + +vendor + Should contain all external dependencies. These should not contain + version numbers in the filename. This information should be available in + the header comment of the file. Library plugins should be prefixed with the + library name. eg the hover intent jQuery plugin would have the filename + *jquery.hover-intent.js*. + +templates + Should be stored in a seperate directory and have the .html + extension. +test + Contains the test runner *index.html*. *vendor* contains all test + dependencies and libraries. *spec* contains the actual test files. Each + test file should be the filename with *-spec* appended. + +JSHint +`````` + +All JavaScript should pass `JSHint`_ before being committed. This can +be installed using `npm` (which is bundled with `node`_) by running: :: + + $ npm -g install jshint + +Each project should include a jshint.json file with appropriate configuration +options for the tool. Most text editors can also be configured to read from +this file. + +.. _node: http://nodejs.org +.. _jshint: http://www.jshint.com + +Documentation +````````````` + +_TODO_ + +Testing +``````` + +_TODO_ + +Best Practices +`````````````` + +Forms +..... + +All forms should work without JavaScript enabled. This means that they must +submit `application/x-www-form-urlencoded` data to the server and receive an appropriate +response. The server should check for the `X-Requested-With: XMLHTTPRequest` +header to determine if the request is an ajax one. If so it can return an +appropriate format, otherwise it should issue a 303 redirect. + +The one exception to this rule is if a form or button is injected with +JavaScript after the page has loaded. It's then not part of the HTML document +and can submit any data format it pleases. + +Ajax +.... + +Ajax requests can be used to improve the experience of submitting forms and +other actions that require server interactions. Nearly all requests will +go through the following states. + +1. User clicks button. +2. JavaScript intercepts the click and disables the button (add `disabled` + attr). +3. A loading indicator is displayed (add class `.loading` to button). +4. The request is made to the server. +5. a) On success the interface is updated. + b) On error a message is displayed to the user if there is no other way to + resolve the issue. +6. The loading indicator is removed. +7. The button is re-enabled. + +Here's a possible example for submitting a search form using jQuery. :: + + jQuery('#search-form').submit(function (event) { + var form = $(this); + var button = form.find('[type=submit]'); + + // Prevent the browser submitting the form. + event.preventDefault(); + + button.prop('disabled', true).addClass('loading'); + + jQuery.ajax({ + type: this.method, + data: form.serialize(), + success: function (results) { + updatePageWithResults(results); + }, + error: function () { + showSearchError('Sorry we were unable to complete this search'); + }, + complete: function () { + button.prop('disabled', false).removeClass('loading'); + } + }); + }); + +This covers possible issues that might arise from submitting the form as well +as providing the user with adequate feedback that the page is doing something. +Disabling the button prevents the form being submitted twice and the error +feedback should hopefully offer a solution for the error that occurred. + +Event Handlers +.............. + +When using event handlers to listen for browser events it's a common +requirement to want to cancel the default browser action. This should be +done by calling the `event.preventDefault()` method: :: + + jQuery('button').click(function (event) { + event.preventDefault(); + }); + +It is also possible to return ``false`` from the callback function. Avoid doing +this as it also calls the ``event.stopPropagation()`` method which prevents the +event from bubbling up the DOM tree. This prevents other handlers listening +for the same event. For example an analytics click handler attached to the +```` element. + +Also jQuery (1.7+) now provides the `.on()`_ and `.off()`_ methods as +alternatives to ``.bind()``, ``.unbind()``, ``.delegate()`` and +``.undelegate()`` and they should be preferred for all tasks. + +.. _.on(): http://api.jquery.com/on/ +.. _.off(): http://api.jquery.com/off/ + +Closures +........ + +_TODO_ + +Templating +.......... + +_TODO_ + +Resources +````````` + +_TODO_ + +{:js: data-code="js"} + - i18n - In html, don't include line breaks within ``

    `` blocks. ie do this: :: From e72fa91427614493ecfab5836a6b5713b4dedfd3 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Wed, 20 Jun 2012 13:36:32 +0100 Subject: [PATCH 007/155] [#2437] Promote the javascript sub-sections so they appear in TOC --- doc/coding-standards.rst | 41 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/doc/coding-standards.rst b/doc/coding-standards.rst index 76948250f8d..0aea1a0f266 100644 --- a/doc/coding-standards.rst +++ b/doc/coding-standards.rst @@ -70,14 +70,11 @@ When merging a feature or bug branch into master: - Use the ``--no-ff`` option in the ``git merge`` command - Add an entry to the ``CHANGELOG`` file -Frontend Coding Standards -========================= - -Javavscript ------------ +Javascript Coding Standards +=========================== Formatting -`````````` +---------- .. _OKFN Coding Standards: http://wiki.okfn.org/Coding_Standards#Javascript .. _idiomatic.js: https://github.com/rwldrn/idiomatic.js/ @@ -94,7 +91,7 @@ exceptions. guide which is recommended by the `OKFN Coding Standards`_. White Space -........... +``````````` Two spaces must be used for indentation at all times. Unlike in idiomatic whitespace must not be used _inside_ parentheses between the parentheses @@ -121,7 +118,7 @@ and their Contents. :: .. note:: See section 2.D.1.1 of idiomatic for more examples of this syntax. Quotes -...... +`````` Single quotes should be used everywhere unless writing JSON or the string contains them. This makes it easier to create strings containing HTML. :: @@ -136,7 +133,7 @@ Object properties need not be quoted unless required by the interpreter. :: }; Variable declarations -..................... +````````````````````` One ``var`` statement must be used per variable assignment. These must be declared at the top of the function in which they are being used. :: @@ -190,7 +187,7 @@ statement. :: } Naming -`````` +------ All properties, functions and methods must use lowercase camelCase: :: @@ -251,7 +248,7 @@ Named functions are generally easier to debug as they appear named in the debugger. Comments -```````` +-------- Comments should be used to explain anything that may be unclear when you return to it in six months time. Single line comments should be used for all inline @@ -271,7 +268,7 @@ comments that do not form part of the documentation. :: } File Structure -`````````````` +-------------- All public JavaScript files should be contained within a _javascript_ directory within the _public_ directory and files should be structured accordingly. :: @@ -318,7 +315,7 @@ test test file should be the filename with *-spec* appended. JSHint -`````` +------ All JavaScript should pass `JSHint`_ before being committed. This can be installed using `npm` (which is bundled with `node`_) by running: :: @@ -333,20 +330,20 @@ this file. .. _jshint: http://www.jshint.com Documentation -````````````` +------------- _TODO_ Testing -``````` +------- _TODO_ Best Practices -`````````````` +-------------- Forms -..... +````` All forms should work without JavaScript enabled. This means that they must submit `application/x-www-form-urlencoded` data to the server and receive an appropriate @@ -359,7 +356,7 @@ JavaScript after the page has loaded. It's then not part of the HTML document and can submit any data format it pleases. Ajax -.... +```` Ajax requests can be used to improve the experience of submitting forms and other actions that require server interactions. Nearly all requests will @@ -408,7 +405,7 @@ Disabling the button prevents the form being submitted twice and the error feedback should hopefully offer a solution for the error that occurred. Event Handlers -.............. +`````````````` When using event handlers to listen for browser events it's a common requirement to want to cancel the default browser action. This should be @@ -432,17 +429,17 @@ alternatives to ``.bind()``, ``.unbind()``, ``.delegate()`` and .. _.off(): http://api.jquery.com/off/ Closures -........ +```````` _TODO_ Templating -.......... +`````````` _TODO_ Resources -````````` +--------- _TODO_ From 9b64dd9975ba30a3fdc0826801e7ea79a183bf1b Mon Sep 17 00:00:00 2001 From: "Sven R. Kunze" Date: Wed, 20 Jun 2012 16:19:07 +0200 Subject: [PATCH 008/155] Unified activity stream implemented unified means activities of the user, of the users being followed by the user and of the datasets being followed by the user --- ckan/controllers/user.py | 6 +--- ckan/logic/action/get.py | 52 +++++++++++++++++++++++++++++- ckan/model/activity.py | 4 +-- ckan/templates/user/dashboard.html | 41 ++++++++--------------- 4 files changed, 68 insertions(+), 35 deletions(-) diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index e2a0060bf99..0f63f8de1f8 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -450,10 +450,6 @@ def dashboard(self, id=None): 'user': c.user or c.author, 'for_view': True} data_dict = {'id':id, 'user_obj':c.userobj} self._setup_template_variables(context, data_dict) - c.followed_users = get_action('followed_user_list')(context, {'id':c.user_dict['id']}) - c.followed_datasets = get_action('followed_dataset_list')(context, {'id':c.user_dict['id']}) + c.dashboard_activity_stream = get_action('dashboard_activity_list_html')(context, {'id':c.user_dict['id']}) return render('user/dashboard.html') - - - diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 8a98950b0d1..c8b0b796576 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1834,6 +1834,7 @@ def user_follower_count(context, data_dict): :param id: the id or name of the user :type id: string + :rtype: int ''' @@ -1849,6 +1850,7 @@ def dataset_follower_count(context, data_dict): :param id: the id or name of the dataset :type id: string + :rtype: int ''' @@ -1877,6 +1879,7 @@ def user_follower_list(context, data_dict): :param id: the id or name of the user :type id: string + :rtype: list of dictionaries ''' @@ -1893,6 +1896,7 @@ def dataset_follower_list(context, data_dict): :param id: the id or name of the dataset :type id: string + :rtype: list of dictionaries ''' @@ -1923,6 +1927,7 @@ def am_following_user(context, data_dict): :param id: the id or name of the user :type id: string + :rtype: boolean ''' @@ -1940,6 +1945,7 @@ def am_following_dataset(context, data_dict): :param id: the id or name of the dataset :type id: string + :rtype: boolean ''' @@ -1952,12 +1958,12 @@ def am_following_dataset(context, data_dict): return _am_following(context, data_dict, context['model'].UserFollowingDataset) - def followed_user_list(context, data_dict): '''Return the list of users that are followed by the given user. :param id: the id or name of the user :type id: string + :rtype: list of dictionaries ''' @@ -1984,6 +1990,7 @@ def followed_dataset_list(context, data_dict): :param id: the id or name of the user :type id: string + :rtype: list of dictionaries ''' @@ -2004,3 +2011,46 @@ def followed_dataset_list(context, data_dict): # Dictize the list of Package objects. return [model_dictize.package_dictize(package, context) for package in packages] + +def dashboard_activity_list(context, data_dict): + '''Return the activity stream of the given user. + + :param id: the id or name of the user + :type id: string + + :rtype: list of dictionaries + + ''' + model = context['model'] + user_id = _get_or_bust(data_dict, 'id') + + activity_query = model.Session.query(model.Activity) + user_query = activity_query; + followed_users_query = activity_query.join(model.UserFollowingUser, model.UserFollowingUser.object_id == model.Activity.user_id) + followed_datasets_query = activity_query.join(model.UserFollowingDataset, model.UserFollowingDataset.object_id == model.Activity.object_id) + + user_query = user_query.filter(model.Activity.user_id==user_id) + followed_users_query = followed_users_query.filter(model.UserFollowingUser.follower_id==user_id) + followed_datasets_query = followed_datasets_query.filter(model.UserFollowingDataset.follower_id==user_id) + + query = user_query.union(followed_users_query).union(followed_datasets_query) + query = query.order_by(_desc(model.Activity.timestamp)) + query = query.limit(15) + activity_objects = query.all() + + return model_dictize.activity_list_dictize(activity_objects, context) + +def dashboard_activity_list_html(context, data_dict): + '''Return the dashboard activity stream of the given user as HTML. + + The activity stream is rendered as a snippet of HTML meant to be included + in an HTML page, i.e. it doesn't have any HTML header or footer. + + :param id: The id or name of the user. + :type id: string + + :rtype: string + + ''' + activity_stream = dashboard_activity_list(context, data_dict) + return _activity_list_to_html(context, activity_stream) diff --git a/ckan/model/activity.py b/ckan/model/activity.py index ff72337f04d..f4b6a7ef715 100644 --- a/ckan/model/activity.py +++ b/ckan/model/activity.py @@ -14,8 +14,8 @@ 'activity', meta.metadata, Column('id', types.UnicodeText, primary_key=True, default=_types.make_uuid), Column('timestamp', types.DateTime), - Column('user_id', types.UnicodeText), - Column('object_id', types.UnicodeText), + Column('user_id', types.UnicodeText, ForeignKey('user.id')), + Column('object_id', types.UnicodeText, ForeignKey('package.id')), Column('revision_id', types.UnicodeText), Column('activity_type', types.UnicodeText), Column('data', _types.JsonDictType), diff --git a/ckan/templates/user/dashboard.html b/ckan/templates/user/dashboard.html index 20681c9577f..74a2199707a 100644 --- a/ckan/templates/user/dashboard.html +++ b/ckan/templates/user/dashboard.html @@ -1,32 +1,19 @@ - - ${c.user} - Dashboard - User - - ${c.user_dict.display_name}'s Dashboard - + xmlns:i18n="http://genshi.edgewall.org/i18n" + xmlns:xi="http://www.w3.org/2001/XInclude" + py:strip=""> -

    XXXX ${len(c.followed_users)} XXXX

    -

    XXXX ${len(c.followed_datasets)} XXXX

    + ${c.user} - Dashboard - User + + ${c.user_dict.display_name}'s Dashboard + -
    -
      -
    • -
        - ${h.linked_user(followed_user['name'], maxlength=20)} -
      -
      • -
    -
      -
    • -
        - ${h.dataset_link(followed_dataset['name'])} -
      -
      • -
    -
    +
    +

    What's going on?

    +
    + ${c.dashboard_activity_stream} +
    +
    - + From 5cefb0980a698045751245b2bed2ccbd51222894 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Wed, 20 Jun 2012 15:19:54 +0100 Subject: [PATCH 009/155] [#2437] Import aron's html style guide. From http://aron.github.com/ckan-style/styleguide/html/ --- doc/coding-standards.rst | 150 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 140 insertions(+), 10 deletions(-) diff --git a/doc/coding-standards.rst b/doc/coding-standards.rst index 0aea1a0f266..f7a0ef6df10 100644 --- a/doc/coding-standards.rst +++ b/doc/coding-standards.rst @@ -441,23 +441,153 @@ _TODO_ Resources --------- -_TODO_ +HTML Coding Standards +===================== + +Formatting +---------- + +All HTML documents must use **two spaces** for indentation and there should be +no trailing whitespace. XHTML syntax must be used (this is more a Genshi +requirement) and all attributes must use double quotes around attributes. :: + + + + +HTML5 elements should be used where appropriate reserving `
    ` and `` +elements for situations where there is no semantic value (such as wrapping +elements to provide styling hooks). + +Doctype and layout +------------------ + +All documents must be using the HTML5 doctype and the `` element should +have a `"lang"` attribute. The `` should also at a minimum include +`"viewport"` and `"charset"` meta tags. :: + + + + + + + Example Site + + + + +Forms +----- + +Form fields must always include a `
 From eba1856526076ddaa5e0ac6667ba0e0a6f8f2b5d Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 27 Jun 2012 12:23:26 +0200 Subject: [PATCH 073/155] [#2592] Replace pyenv with ~/pyenv in docs --- doc/install-from-source.rst | 21 ++++++++++----------- doc/test.rst | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/doc/install-from-source.rst b/doc/install-from-source.rst index 88ac3d2b1dc..4e14d50a324 100644 --- a/doc/install-from-source.rst +++ b/doc/install-from-source.rst @@ -46,12 +46,11 @@ OpenJDK 6 JDK `The Java Development Kit Date: Wed, 27 Jun 2012 13:23:12 +0100 Subject: [PATCH 074/155] [#2592] stop tip being quoted --- doc/install-from-source.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/install-from-source.rst b/doc/install-from-source.rst index 4e14d50a324..64bdc79a3a5 100644 --- a/doc/install-from-source.rst +++ b/doc/install-from-source.rst @@ -80,7 +80,7 @@ the CKAN install. Next you'll need to create a database user if one doesn't already exist. - .. tip :: +.. tip :: If you choose a database name, user or password which are different from the example values suggested below then you'll need to change the From de5c8549995dd178ea910757750876c677a2ff04 Mon Sep 17 00:00:00 2001 From: Toby Date: Wed, 27 Jun 2012 15:18:19 +0100 Subject: [PATCH 075/155] [#2592] add changelog comment --- CHANGELOG.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 21a87c73948..3ac54bb6a6e 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,12 @@ CKAN CHANGELOG ++++++++++++++ +v1.8 +==== + +* requirements have been updated see doc/install-from-source.rst +users will need to do a new pip install (#2592) + v1.7.1 2012-06-20 ================= From 152cf537ca09ebad8e8d9fcb6bdf37f8b36feae1 Mon Sep 17 00:00:00 2001 From: John Glover Date: Wed, 27 Jun 2012 15:52:18 +0100 Subject: [PATCH 076/155] [1732] Fix tag facet list when using vocab tags. Removes vocab tags from tag list when indexing. Adds vocab tags to their own dynamic field called vocab_. --- ckan/config/solr/CHANGELOG.txt | 5 ++ ckan/config/solr/schema-1.4.xml | 2 + ckan/lib/search/index.py | 26 +++++++-- ckan/tests/logic/test_tag_vocab.py | 85 +++++++++++++++++++++++++++--- 4 files changed, 105 insertions(+), 13 deletions(-) diff --git a/ckan/config/solr/CHANGELOG.txt b/ckan/config/solr/CHANGELOG.txt index 1e4e67f6ff6..735d6a8dc6c 100644 --- a/ckan/config/solr/CHANGELOG.txt +++ b/ckan/config/solr/CHANGELOG.txt @@ -1,6 +1,11 @@ CKAN SOLR schemas changelog =========================== +v1.4 - (ckan>=1.8) +------------------ +* Add vocab_* dynamic field so it is possible to facet by vocabulary tags +* Add copyField for text with source vocab_* + v1.4 - (ckan>=1.7) -------------------- * Add Ascii folding filter to text fields. diff --git a/ckan/config/solr/schema-1.4.xml b/ckan/config/solr/schema-1.4.xml index 0409e71b14b..d98b9c56f5c 100644 --- a/ckan/config/solr/schema-1.4.xml +++ b/ckan/config/solr/schema-1.4.xml @@ -153,6 +153,7 @@ + @@ -165,6 +166,7 @@ + diff --git a/ckan/lib/search/index.py b/ckan/lib/search/index.py index e9129009947..09ec2eec913 100644 --- a/ckan/lib/search/index.py +++ b/ckan/lib/search/index.py @@ -1,7 +1,6 @@ import socket import string import logging -import itertools import collections import json @@ -14,6 +13,7 @@ import ckan.model as model from ckan.plugins import (PluginImplementations, IPackageController) +import ckan.logic as logic log = logging.getLogger(__name__) @@ -122,10 +122,27 @@ def index_package(self, pkg_dict): pkg_dict[key] = value pkg_dict.pop('extras', None) - #Add tags and groups + # add tags, removing vocab tags from 'tags' list and adding them as + # vocab_ so that they can be used in facets + non_vocab_tag_names = [] tags = pkg_dict.pop('tags', []) - pkg_dict['tags'] = [tag['name'] for tag in tags] - + context = {'model': model} + + for tag in tags: + if tag.get('vocabulary_id'): + data = {'id': tag['vocabulary_id']} + vocab = logic.get_action('vocabulary_show')(context, data) + key = u'vocab_%s' % vocab['name'] + if key in pkg_dict: + pkg_dict[key].append(tag['name']) + else: + pkg_dict[key] = [tag['name']] + else: + non_vocab_tag_names.append(tag['name']) + + pkg_dict['tags'] = non_vocab_tag_names + + # add groups groups = pkg_dict.pop('groups', []) # Capacity is different to the default only if using organizations @@ -197,7 +214,6 @@ def index_package(self, pkg_dict): import hashlib pkg_dict['index_id'] = hashlib.md5('%s%s' % (pkg_dict['id'],config.get('ckan.site_id'))).hexdigest() - for item in PluginImplementations(IPackageController): pkg_dict = item.before_index(pkg_dict) diff --git a/ckan/tests/logic/test_tag_vocab.py b/ckan/tests/logic/test_tag_vocab.py index 4500ea69a7f..9f806e3ce54 100644 --- a/ckan/tests/logic/test_tag_vocab.py +++ b/ckan/tests/logic/test_tag_vocab.py @@ -1,13 +1,17 @@ -from ckan import model -from ckan.logic.converters import convert_to_tags, convert_from_tags, free_tags_only -from ckan.lib.navl.dictization_functions import unflatten +import ckan.model as model +import ckan.logic as logic +import ckan.logic.converters as converters +import ckan.lib.navl.dictization_functions as df +import ckan.tests as tests +import ckan.lib.create_test_data as ctd TEST_VOCAB_NAME = 'test-vocab' + class TestConverters(object): @classmethod def setup_class(cls): - cls.vocab = model.Vocabulary(TEST_VOCAB_NAME) + cls.vocab = model.Vocabulary(TEST_VOCAB_NAME) model.Session.add(cls.vocab) model.Session.commit() vocab_tag_1 = model.Tag('tag1', cls.vocab.id) @@ -27,11 +31,11 @@ def convert(tag_string, vocab): data = {key: tag_string} errors = [] context = {'model': model, 'session': model.Session} - convert_to_tags(vocab)(key, data, errors, context) + converters.convert_to_tags(vocab)(key, data, errors, context) del data[key] return data - data = unflatten(convert(['tag1', 'tag2'], 'test-vocab')) + data = df.unflatten(convert(['tag1', 'tag2'], 'test-vocab')) for tag in data['tags']: assert tag['name'] in ['tag1', 'tag2'], tag['name'] assert tag['vocabulary_id'] == self.vocab.id, tag['vocabulary_id'] @@ -44,7 +48,7 @@ def test_convert_from_tags(self): } errors = [] context = {'model': model, 'session': model.Session} - convert_from_tags(self.vocab.name)(key, data, errors, context) + converters.convert_from_tags(self.vocab.name)(key, data, errors, context) assert 'tag1' in data['tags'] assert 'tag2' in data['tags'] @@ -58,8 +62,73 @@ def test_free_tags_only(self): } errors = [] context = {'model': model, 'session': model.Session} - free_tags_only(key, data, errors, context) + converters.free_tags_only(key, data, errors, context) assert len(data) == 2 assert ('tags', 1, 'vocabulary_id') in data.keys() assert ('tags', 1, '__extras') in data.keys() + +class TestVocabFacets(object): + @classmethod + def setup_class(cls): + if not tests.is_search_supported(): + raise tests.SkipTest("Search not supported") + tests.setup_test_search_index() + + ctd.CreateTestData.create() + + model.repo.new_revision() + cls.vocab = model.Vocabulary(TEST_VOCAB_NAME) + model.Session.add(cls.vocab) + model.Session.commit() + + vocab_tag_1 = model.Tag('tag1', cls.vocab.id) + vocab_tag_2 = model.Tag('tag2', cls.vocab.id) + model.Session.add(vocab_tag_1) + model.Session.add(vocab_tag_2) + + pkg = model.Package.get('warandpeace') + pkg_tag1 = model.PackageTag(pkg, vocab_tag_1) + pkg_tag2 = model.PackageTag(pkg, vocab_tag_2) + model.Session.add(pkg_tag1) + model.Session.add(pkg_tag2) + + model.Session.commit() + model.Session.remove() + + @classmethod + def teardown_class(cls): + model.repo.rebuild_db() + + def test_vocab_facets(self): + vocab_facet = 'vocab_%s' % TEST_VOCAB_NAME + + context = {'model': model, 'session': model.Session} + data = { + 'q': 'warandpeace', + 'facet': 'true', + 'facet.field': ['groups', 'tags', vocab_facet], + 'facet.limit': '50', + 'facet.mincount': 1, + } + + result = logic.get_action('package_search')(context, data) + facets = result['search_facets'] + facet_tags = [t['name'] for t in facets['tags']['items']] + assert len(facet_tags) + + # make sure vocab tags are not in 'tags' facet + assert not 'tag1' in facet_tags + assert not 'tag2' in facet_tags + + # make sure vocab tags are in vocab_ facet + vocab_facet_tags = [t['name'] for t in facets[vocab_facet]['items']] + assert 'tag1' in vocab_facet_tags + assert 'tag2' in vocab_facet_tags + + def test_vocab_facets_searchable(self): + context = {'model': model, 'session': model.Session} + data = {'q': 'tag1', 'facet': 'false'} + result = logic.get_action('package_search')(context, data) + assert result['count'] == 1 + assert result['results'][0]['name'] == 'warandpeace' From a03c9605ab7d7ecbae90b1d40f0f052a73b0f410 Mon Sep 17 00:00:00 2001 From: "Sven R. Kunze" Date: Wed, 27 Jun 2012 17:00:01 +0200 Subject: [PATCH 077/155] Dashboard now available through API; API tests modified --- ckan/controllers/api.py | 1 + ckan/lib/helpers.py | 2 +- ckan/logic/action/get.py | 2 +- ckan/tests/functional/api/test_activity.py | 86 ++++++++++++++++++++++ 4 files changed, 89 insertions(+), 2 deletions(-) diff --git a/ckan/controllers/api.py b/ckan/controllers/api.py index 723d4bbf6cc..66754fdb1f2 100644 --- a/ckan/controllers/api.py +++ b/ckan/controllers/api.py @@ -251,6 +251,7 @@ def list(self, ver=None, register=None, subregister=None, id=None): ('dataset', 'activity'): 'package_activity_list', ('group', 'activity'): 'group_activity_list', ('user', 'activity'): 'user_activity_list', + ('user', 'dashboard_activity'): 'dashboard_activity_list', ('activity', 'details'): 'activity_detail_list', } diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 6a3d520b4fe..114efb0a78e 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -883,7 +883,7 @@ def dashboard_activity_stream(user_id): ''' import ckan.logic as logic context = {'model' : model, 'session':model.Session, 'user':c.user} - return logic.get_action('dashboard_activity_list_html')(context, {'user_id': user_id}) + return logic.get_action('dashboard_activity_list_html')(context, {'id': user_id}) # these are the functions that will end up in `h` template helpers diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 79ff3edabd6..a8a6fa9c506 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -2054,7 +2054,7 @@ def dashboard_activity_list(context, data_dict): ''' model = context['model'] - user_id = _get_or_bust(data_dict, 'user_id') + user_id = _get_or_bust(data_dict, 'id') activity_query = model.Session.query(model.Activity) user_query = activity_query; diff --git a/ckan/tests/functional/api/test_activity.py b/ckan/tests/functional/api/test_activity.py index 64ff6a2f743..4814b064d0e 100644 --- a/ckan/tests/functional/api/test_activity.py +++ b/ckan/tests/functional/api/test_activity.py @@ -96,14 +96,51 @@ def setup_class(self): ckan.tests.CreateTestData.create() self.sysadmin_user = model.User.get('testsysadmin') self.normal_user = model.User.get('annafan') + self.follower = model.User.get('tester') self.warandpeace = model.Package.get('warandpeace') self.annakarenina = model.Package.get('annakarenina') self.app = paste.fixture.TestApp(pylonsapp) + # Make follower follow everything else. + params = {'id': 'testsysadmin'} + extra_environ = {'Authorization': str(self.follower.apikey)} + response = self.app.post('/api/action/follow_user', + params=json.dumps(params), extra_environ=extra_environ).json + assert response['success'] is True + params = {'id': 'annafan'} + extra_environ = {'Authorization': str(self.follower.apikey)} + response = self.app.post('/api/action/follow_user', + params=json.dumps(params), extra_environ=extra_environ).json + assert response['success'] is True + params = {'id': 'warandpeace'} + extra_environ = {'Authorization': str(self.follower.apikey)} + response = self.app.post('/api/action/follow_dataset', + params=json.dumps(params), extra_environ=extra_environ).json + assert response['success'] is True + params = {'id': 'annakarenina'} + extra_environ = {'Authorization': str(self.follower.apikey)} + response = self.app.post('/api/action/follow_dataset', + params=json.dumps(params), extra_environ=extra_environ).json + assert response['success'] is True + + self.followees = \ + [ + self.sysadmin_user.id, + self.normal_user.id, + self.follower.id, + self.warandpeace.id, + self.annakarenina.id + ] + + @classmethod def teardown_class(self): model.repo.rebuild_db() + def dashboard_activity_stream(self, user_id): + response = self.app.get("/api/2/rest/user/%s/dashboard_activity" % user_id) + return json.loads(response.body) + def user_activity_stream(self, user_id): response = self.app.get("/api/2/rest/user/%s/activity" % user_id) return json.loads(response.body) @@ -145,9 +182,25 @@ def record_details(self, user_id, package_id=None, group_id=None): details['recently changed datasets stream'] = \ self.recently_changed_datasets_stream() + details['follower dashboard activity stream'] = \ + self.dashboard_activity_stream(self.follower.id) + details['time'] = datetime.datetime.now() return details + def check_dashboard( + self, + before, after, wanted_difference, + potential_followees): + difference = find_new_activities( + before['follower dashboard activity stream'], + after['follower dashboard activity stream']) + if any(potential_followee in self.followees for potential_followee in potential_followees): + assert difference == wanted_difference + else: + assert len(difference) == 0 + + def _create_package(self, user, name=None): if user: user_name = user.name @@ -189,6 +242,8 @@ def _create_package(self, user, name=None): after['recently changed datasets stream']) \ == user_new_activities + self.check_dashboard(before, after, user_new_activities, [user_id]) + # Check that the new activity has the right attributes. assert activity['object_id'] == package_created['id'], \ str(activity['object_id']) @@ -297,6 +352,8 @@ def _add_resource(self, package, user): after['recently changed datasets stream']) \ == user_new_activities + self.check_dashboard(before, after, user_new_activities, [user_id, package.id]) + # Check that the new activity has the right attributes. assert activity['object_id'] == updated_package['id'], \ str(activity['object_id']) @@ -386,6 +443,8 @@ def _delete_extra(self, package_dict, user): after['recently changed datasets stream']) \ == user_new_activities + self.check_dashboard(before, after, user_new_activities, [user_id, package_dict['id']]) + # Check that the new activity has the right attributes. assert activity['object_id'] == updated_package['id'], \ str(activity['object_id']) @@ -480,6 +539,8 @@ def _update_extra(self, package_dict, user): after['recently changed datasets stream']) \ == user_new_activities + self.check_dashboard(before, after, user_new_activities, [user_id, package_dict['id']]) + # Check that the new activity has the right attributes. assert activity['object_id'] == updated_package['id'], \ str(activity['object_id']) @@ -570,6 +631,8 @@ def _add_extra(self, package_dict, user, key=None): after['recently changed datasets stream']) \ == user_new_activities + self.check_dashboard(before, after, user_new_activities, [user_id, package_dict['id']]) + # Check that the new activity has the right attributes. assert activity['object_id'] == updated_package['id'], \ str(activity['object_id']) @@ -628,6 +691,8 @@ def _create_activity(self, user, package, params): after['package activity stream'])) assert pkg_new_activities == user_new_activities + self.check_dashboard(before, after, user_new_activities, [user.id, package.id]) + # Check that the new activity has the right attributes. assert activity['object_id'] == params['object_id'], ( str(activity['object_id'])) @@ -671,6 +736,8 @@ def _delete_group(self, group, user): after['group activity stream']) == new_activities, ("The same " "activity should also appear in the group's activity stream.") + self.check_dashboard(before, after, new_activities, [user.id]) + # Check that the new activity has the right attributes. assert activity['object_id'] == group.id, str(activity['object_id']) assert activity['user_id'] == user.id, str(activity['user_id']) @@ -712,6 +779,9 @@ def _update_group(self, group, user): after['group activity stream']) == new_activities, ("The same " "activity should also appear in the group's activity stream.") + self.check_dashboard(before, after, new_activities, [user.id]) + + # Check that the new activity has the right attributes. assert activity['object_id'] == group.id, str(activity['object_id']) assert activity['user_id'] == user.id, str(activity['user_id']) @@ -758,6 +828,8 @@ def _update_user(self, user): "the user's activity stream, but found %i" % len(new_activities)) activity = new_activities[0] + self.check_dashboard(before, after, new_activities, [user.id]) + # Check that the new activity has the right attributes. assert activity['object_id'] == user.id, str(activity['object_id']) assert activity['user_id'] == user.id, str(activity['user_id']) @@ -822,6 +894,8 @@ def _delete_resources(self, package): after['recently changed datasets stream']) \ == user_new_activities + self.check_dashboard(before, after, user_new_activities, [package.id]) + # Check that the new activity has the right attributes. assert activity['object_id'] == package.id, ( str(activity['object_id'])) @@ -905,6 +979,8 @@ def _update_package(self, package, user): after['recently changed datasets stream']) \ == user_new_activities + self.check_dashboard(before, after, user_new_activities, [user_id, package.id]) + # Check that the new activity has the right attributes. assert activity['object_id'] == package.id, ( str(activity['object_id'])) @@ -981,6 +1057,8 @@ def _update_resource(self, package, resource, user): after['recently changed datasets stream']) \ == user_new_activities + self.check_dashboard(before, after, user_new_activities, [user_id, package.id]) + # Check that the new activity has the right attributes. assert activity['object_id'] == package.id, ( str(activity['object_id'])) @@ -1049,6 +1127,8 @@ def _delete_package(self, package): after['recently changed datasets stream']) \ == user_new_activities + self.check_dashboard(before, after, user_new_activities, [self.sysadmin_user.id, package.id]) + # Check that the new activity has the right attributes. assert activity['object_id'] == package.id, ( str(activity['object_id'])) @@ -1160,6 +1240,8 @@ def test_01_remove_tag(self): after['recently changed datasets stream']) \ == user_new_activities + self.check_dashboard(before, after, user_new_activities, [user.id, pkg_dict['id']]) + # Check that the new activity has the right attributes. assert activity['object_id'] == pkg_dict['id'], ( str(activity['object_id'])) @@ -1416,6 +1498,8 @@ def test_create_group(self): assert after['group activity stream'] == new_activities, ("The same " "activity should also appear in the group's activity stream.") + self.check_dashboard(before, after, new_activities, [user.id]) + # Check that the new activity has the right attributes. assert activity['object_id'] == group_created['id'], \ str(activity['object_id']) @@ -1496,6 +1580,8 @@ def test_add_tag(self): after['recently changed datasets stream']) \ == user_new_activities + self.check_dashboard(before, after, user_new_activities, [user.id, pkg_dict['id']]) + # Check that the new activity has the right attributes. assert activity['object_id'] == pkg_dict['id'], ( str(activity['object_id'])) From 1c3361322b6be982615ac793db2b2fc1a456585a Mon Sep 17 00:00:00 2001 From: "Sven R. Kunze" Date: Wed, 27 Jun 2012 17:45:27 +0200 Subject: [PATCH 078/155] brief test for dashboard activtities --- ckan/tests/functional/test_activity.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ckan/tests/functional/test_activity.py b/ckan/tests/functional/test_activity.py index b7f59724ac2..fccdfb42f93 100644 --- a/ckan/tests/functional/test_activity.py +++ b/ckan/tests/functional/test_activity.py @@ -25,8 +25,9 @@ def setup(cls): def teardown(cls): ckan.model.repo.rebuild_db() - def test_activity(self): - """Test activity streams HTML rendering.""" + + def test_user_activity(self): + """Test user activity streams HTML rendering.""" # Register a new user. user_dict = {'name': 'billybeane', @@ -208,3 +209,10 @@ def test_activity(self): result = self.app.get(offset, status=200) assert result.body.count('
') \ == 15 + + # The latest 15 should also appear on the dashboard + offset = url_for(controller='user', action='dashboard') + params = {'id': user['id']} + extra_environ = {'Authorization': str(self.sysadmin_user.apikey)} + response = self.app.post(offset, params=params, extra_environ=extra_environ, status=200) + assert result.body.count('
') == 15 From bad89a82900b154a86606b13d219d81e6893fbe0 Mon Sep 17 00:00:00 2001 From: "Sven R. Kunze" Date: Wed, 27 Jun 2012 18:19:01 +0200 Subject: [PATCH 079/155] user encouragement added --- ckan/templates/user/dashboard.html | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ckan/templates/user/dashboard.html b/ckan/templates/user/dashboard.html index 2d20308f987..289d6516769 100644 --- a/ckan/templates/user/dashboard.html +++ b/ckan/templates/user/dashboard.html @@ -21,5 +21,16 @@

What's going on?

+
+

Nothing new on CKAN?

+

So, why don't you ...

+ +
+ From bd817df3cda3b0fc2e7a20d0b4808c022215cf50 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 27 Jun 2012 17:53:55 +0100 Subject: [PATCH 080/155] [#2389] Try to make the data explorer fail with more dignity Adds some checks for DataStore and DataProxy backends. Recline does not make easy to catch exceptions and errors while initializing, but hopefully this will cover the most common cases. Note: includes a small patch on recline.js, which will be ported upstream. --- ckan/public/scripts/application.js | 39 +++++++++++++++++-- ckan/public/scripts/vendor/recline/recline.js | 5 +++ ckan/templates/js_strings.html | 6 ++- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/ckan/public/scripts/application.js b/ckan/public/scripts/application.js index 11202ad634d..0eb88162420 100644 --- a/ckan/public/scripts/application.js +++ b/ckan/public/scripts/application.js @@ -1553,6 +1553,14 @@ CKAN.DataPreview = function ($, my) { my.loadPreviewDialog = function(resourceData) { my.$dialog.html('

Loading ...

'); + function showError(msg){ + msg = msg || CKAN.Strings.errorLoadingPreview; + return $('#ckanext-datapreview') + .append('
') + .addClass('alert alert-error fade in') + .html(msg); + } + function initializeDataExplorer(dataset) { var views = [ { @@ -1586,6 +1594,7 @@ CKAN.DataPreview = function ($, my) { } }); + // ----------------------------- // Setup the Embed modal dialog. // ----------------------------- @@ -1665,14 +1674,38 @@ CKAN.DataPreview = function ($, my) { if (resourceData.webstore_url) { resourceData.elasticsearch_url = '/api/data/' + resourceData.id; var dataset = new recline.Model.Dataset(resourceData, 'elasticsearch'); - initializeDataExplorer(dataset); + var errorMsg = CKAN.Strings.errorLoadingPreview + ': ' + CKAN.Strings.errorDataStore; + dataset.fetch() + .done(function(dataset){ + initializeDataExplorer(dataset); + }) + .fail(function(error){ + if (error.message) errorMsg += ' (' + error.message + ')'; + showError(errorMsg); + }); + } else if (resourceData.formatNormalized in {'csv': '', 'xls': ''}) { // set format as this is used by Recline in setting format for DataProxy resourceData.format = resourceData.formatNormalized; var dataset = new recline.Model.Dataset(resourceData, 'dataproxy'); - initializeDataExplorer(dataset); - $('.recline-query-editor .text-query').hide(); + var errorMsg = CKAN.Strings.errorLoadingPreview + ': ' +CKAN.Strings.errorDataProxy; + dataset.fetch() + .done(function(dataset){ + + dataset.bind('query:fail', function(error) { + $('#ckanext-datapreview .data-view-container').hide(); + $('#ckanext-datapreview .header').hide(); + $('.preview-header .btn').hide(); + }); + + initializeDataExplorer(dataset); + $('.recline-query-editor .text-query').hide(); + }) + .fail(function(error){ + if (error.message) errorMsg += ' (' + error.message + ')'; + showError(errorMsg); + }); } else if (resourceData.formatNormalized in { 'rdf+xml': '', diff --git a/ckan/public/scripts/vendor/recline/recline.js b/ckan/public/scripts/vendor/recline/recline.js index 271e9c54fb9..39ca7c9c737 100644 --- a/ckan/public/scripts/vendor/recline/recline.js +++ b/ckan/public/scripts/vendor/recline/recline.js @@ -3177,6 +3177,11 @@ this.recline.Backend = this.recline.Backend || {}; var dfd = $.Deferred(); this._wrapInTimeout(jqxhr).done(function(schema) { // only one top level key in ES = the type so we can ignore it + // CKAN + if (!schema){ + dfd.reject({'message':'Elastic Search did not return a mapping'}); + return; + } var key = _.keys(schema)[0]; var fieldData = _.map(schema[key].properties, function(dict, fieldName) { dict.id = fieldName; diff --git a/ckan/templates/js_strings.html b/ckan/templates/js_strings.html index 61db8cf8e29..77314fb993f 100644 --- a/ckan/templates/js_strings.html +++ b/ckan/templates/js_strings.html @@ -66,7 +66,11 @@ deleteResource = _('Delete Resource'), youCanUseMarkdown = _('You can use %aMarkdown formatting%b here.'), datesAreInISO = _('Dates are in %aISO Format%b — eg. %c2012-12-25%d or %c2010-05-31T14:30%d.'), - dataFileUploaded = _('Data File (Uploaded)') + dataFileUploaded = _('Data File (Uploaded)'), + errorLoadingPreview = _('Could not load preview'), + errorDataProxy = _('DataProxy returned an error'), + errorDataStore = _('DataStore returned an error') + ), indent=4)} From 7f59a58e15dfc1d2498534ef0d474cc7ea08d2b9 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 27 Jun 2012 18:13:57 +0100 Subject: [PATCH 081/155] Update datastore docs --- doc/datastore.rst | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/doc/datastore.rst b/doc/datastore.rst index 15fcd8fe35c..9f3de6a919c 100644 --- a/doc/datastore.rst +++ b/doc/datastore.rst @@ -66,19 +66,41 @@ Please see the ElasticSearch_ documentation. 2. Configure Nginx ------------------ -You must add to your Nginx CKAN site entry the following:: - - location /elastic/ { - internal; - # location of elastic search - proxy_pass http://0.0.0.0:9200/; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +As previously mentioned, Nginx will be used on top of CKAN to forward +requests to Elastic Search. CKAN will still be served by Apache or the +development server (Paster), but all requests will be forwarded to it +by Ngnix. + +This is an example of an Nginx configuration file. Note the two locations +defined, `/` will point to the server running CKAN (Apache or Paster), and +`/elastic/` to the Elastic Search instance:: + + server { + listen 80 default; + server_name localhost; + + access_log /var/log/nginx/localhost.access.log; + + location / { + # location of apache or ckan under paster + proxy_pass http://127.0.0.1:8080; + proxy_set_header Host $host; + } + location /elastic/ { + internal; + # location of elastic search + proxy_pass http://:127.0.0.1:9200/; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } } .. note:: update the proxy_pass field value to point to your ElasticSearch instance (if it is not localhost and default port). +Remember that after setting up Nginx, you need to access CKAN via its port +(80), not the Apache or Paster (5000) one, otherwise the DataStore won't work. + 3. Enable datastore features in CKAN ------------------------------------ From 57edefe385abf276c6874e6f7ce30b7621eca122 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 28 Jun 2012 11:14:15 +0200 Subject: [PATCH 082/155] Remove print statements from test_user.py --- ckan/tests/functional/test_user.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ckan/tests/functional/test_user.py b/ckan/tests/functional/test_user.py index 1ee29522b22..8206d317676 100644 --- a/ckan/tests/functional/test_user.py +++ b/ckan/tests/functional/test_user.py @@ -3,7 +3,6 @@ from pylons import config import hashlib -from pprint import pprint from ckan.tests import search_related, CreateTestData from ckan.tests.html_check import HtmlCheckMethods from ckan.tests.pylons_controller import PylonsTestCase @@ -192,7 +191,6 @@ def test_login(self): res.header('Location').startswith('/user/dashboard') res = res.follow() assert_equal(res.status, 200) - print res.body assert 'testlogin is now logged in' in res.body assert 'checkpoint:my-dashboard' in res.body @@ -208,9 +206,7 @@ def test_login(self): assert 'testlogin!userid_type:unicode' in cookie, cookie # navigate to another page and check username still displayed - print res.body res = res.click('Search') - print res assert 'testlogin' in res.body, res.body def test_login_remembered(self): @@ -948,7 +944,6 @@ def test_reset_user_password_link(self): create_reset_key(model.User.by_name(u'bob')) reset_password_link = get_reset_link(model.User.by_name(u'bob')) offset = reset_password_link.replace('http://test.ckan.net', '') - print offset res = self.app.get(offset) # Reset password form From 3cc2f81745260c3569991758452adf32b5fb0289 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 28 Jun 2012 11:25:30 +0200 Subject: [PATCH 083/155] [#2347] Don't add foreign keys to activity model Don't want to have to do a db migrate for this branch --- ckan/model/activity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/model/activity.py b/ckan/model/activity.py index f4b6a7ef715..ff72337f04d 100644 --- a/ckan/model/activity.py +++ b/ckan/model/activity.py @@ -14,8 +14,8 @@ 'activity', meta.metadata, Column('id', types.UnicodeText, primary_key=True, default=_types.make_uuid), Column('timestamp', types.DateTime), - Column('user_id', types.UnicodeText, ForeignKey('user.id')), - Column('object_id', types.UnicodeText, ForeignKey('package.id')), + Column('user_id', types.UnicodeText), + Column('object_id', types.UnicodeText), Column('revision_id', types.UnicodeText), Column('activity_type', types.UnicodeText), Column('data', _types.JsonDictType), From 59b3297bae4461b47037071db26ec80e18eef81a Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 28 Jun 2012 11:42:38 +0200 Subject: [PATCH 084/155] [#2347] Remove an unnecessary route, fix link --- ckan/config/routing.py | 1 - ckan/templates/user/layout.html | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ckan/config/routing.py b/ckan/config/routing.py index 5b372a65804..b529d90eed7 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -253,7 +253,6 @@ def make_map(): # Note: openid users have slashes in their ids, so need the wildcard # in the route. m.connect('/user/dashboard', action='dashboard') - m.connect('/user/read', action='read') m.connect('/user/followers/{id:.*}', action='followers') m.connect('/user/edit/{id:.*}', action='edit') m.connect('/user/reset/{id:.*}', action='perform_reset') diff --git a/ckan/templates/user/layout.html b/ckan/templates/user/layout.html index b4ec82523ed..e733d4df336 100644 --- a/ckan/templates/user/layout.html +++ b/ckan/templates/user/layout.html @@ -9,7 +9,7 @@
  • Dashboard
    • -
  • My Profile
    • +
  • My Profile
  • Edit Profile
  • Log out
  • From c72c53c28ac32ed64159df263c6acb76238740bd Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 28 Jun 2012 11:47:58 +0200 Subject: [PATCH 085/155] [#2305] Remove some duplicated css --- ckan/public/css/style.css | 34 ++++++---------------------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/ckan/public/css/style.css b/ckan/public/css/style.css index 3812d6bd859..6df5b558884 100644 --- a/ckan/public/css/style.css +++ b/ckan/public/css/style.css @@ -615,31 +615,9 @@ ul.userlist .badge { margin-top: 5px; } -/* ================== */ -/* = Dashboard page = */ -/* ================== */ - -.user.dashboard .page_heading { - font-weight: bold; -} - -.user.dashboard .page_heading img.gravatar { - padding: 2px; - border: solid 1px #ddd; - vertical-align: middle; - margin-right: 5px; - margin-top: -3px; -} - -.user.dashboard .page_heading .fullname { - font-weight: normal; - color: #999; -} - - -/* ================== */ -/* = User Read page = */ -/* ================== */ +/* ================================= */ +/* = User Read and Dashboard pages = */ +/* ================================= */ body.user.read #sidebar { display: none; } body.user.read #content { @@ -647,11 +625,11 @@ body.user.read #content { width: 950px; } -.user.read .page_heading { +.user.read .page_heading, .user.dashboard .page_heading { font-weight: bold; } -.user.read .page_heading img.gravatar { +.user.read .page_heading img.gravatar, .user.dashboard .page_heading img.gravatar { padding: 2px; border: solid 1px #ddd; vertical-align: middle; @@ -659,7 +637,7 @@ body.user.read #content { margin-top: -3px; } -.user.read .page_heading .fullname { +.user.read .page_heading .fullname, .user.dashboard .page_heading .fullname { font-weight: normal; color: #999; } From 5e165b9e38f716b38907e3ac3e162f62c221d0d9 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 28 Jun 2012 10:51:29 +0100 Subject: [PATCH 086/155] [#2389] Fix silly errors on datastore tests --- ckan/public/scripts/application.js | 4 ++-- ckan/tests/functional/test_datastore.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ckan/public/scripts/application.js b/ckan/public/scripts/application.js index 0eb88162420..d49af47cf4a 100644 --- a/ckan/public/scripts/application.js +++ b/ckan/public/scripts/application.js @@ -1001,7 +1001,7 @@ CKAN.Utils = function($, my) { input_box.attr('name', new_name); input_box.attr('id', new_name); - + var $new = $('

    '); $new.append($('').attr('name', old_name).val(ui.item.value)); $new.append(' '); @@ -1447,7 +1447,7 @@ CKAN.Utils = function($, my) { }, }); }; - + // This only needs to happen on dataset pages, but it doesn't seem to do // any harm to call it anyway. $('#user_follow_button').on('click', followButtonClicked); diff --git a/ckan/tests/functional/test_datastore.py b/ckan/tests/functional/test_datastore.py index 27a85e0211c..2fb2d005460 100644 --- a/ckan/tests/functional/test_datastore.py +++ b/ckan/tests/functional/test_datastore.py @@ -76,9 +76,9 @@ def _forward_request(self, res, data=None): assert elastic_search_offset if data: - res = requests.post('http: //%s%s' % (self.elastic_search_host, elastic_search_offset), data=json.dumps(data)) + res = requests.post('http://%s%s' % (self.elastic_search_host, elastic_search_offset), data=json.dumps(data)) else: - res = requests.get('http: //%s%s' % (self.elastic_search_host, elastic_search_offset)) + res = requests.get('http://%s%s' % (self.elastic_search_host, elastic_search_offset)) return res @@ -111,12 +111,12 @@ def setup_class(cls): # Check if Elastic Search is available, otherwise skip tests try: - requests.get('http: //%s/_status' % ELASTIC_SEARCH_HOST, timeout=2) + requests.get('http://%s/_status' % ELASTIC_SEARCH_HOST, timeout=2) except (requests.exceptions.RequestException, socket.timeout), e: raise SkipTest('Could not reach Elastic Search, skipping... (%r)' % e) # Push dummy data to create the test index - requests.put('http: //%s/ckan-%s/' % (ELASTIC_SEARCH_HOST, config.get('ckan.site_id'))) + requests.put('http://%s/ckan-%s/' % (ELASTIC_SEARCH_HOST, config.get('ckan.site_id'))) model.repo.init_db() CreateTestData.create() @@ -125,7 +125,7 @@ def setup_class(cls): def teardown_class(self): # Delete the test index on ES - requests.delete('http: //%s/ckan-%s/' % (ELASTIC_SEARCH_HOST, config.get('ckan.site_id'))) + requests.delete('http://%s/ckan-%s/' % (ELASTIC_SEARCH_HOST, config.get('ckan.site_id'))) model.repo.rebuild_db() From 06ac119c0565001e9a11f9483e2aab6032fa3edf Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 28 Jun 2012 12:03:32 +0200 Subject: [PATCH 087/155] Add follow feature to CHANGELOG --- CHANGELOG.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 3ac54bb6a6e..c5d44bb1f7c 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -6,6 +6,13 @@ v1.8 * requirements have been updated see doc/install-from-source.rst users will need to do a new pip install (#2592) +* New 'follow' feature. You'll now see a 'Followers' tab on user and dataset + pages, where you can see how many users are following that user or dataset. + If you're logged in, you'll see a 'Follow' button on the pages of datasets + and other users that you can click to follow them. Also when logged in, if + you go to your own user page you'll see a new 'Dashboard' tab where you can + see an activity stream from of all the users and datasets that you're + following. v1.7.1 2012-06-20 ================= From f84b110e38bae47a4cfb05ea062c6d52ed6f144a Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 28 Jun 2012 12:08:48 +0200 Subject: [PATCH 088/155] [#2305] Add follow api to changelog --- CHANGELOG.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index c5d44bb1f7c..adaca8a3cf4 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -12,7 +12,8 @@ users will need to do a new pip install (#2592) and other users that you can click to follow them. Also when logged in, if you go to your own user page you'll see a new 'Dashboard' tab where you can see an activity stream from of all the users and datasets that you're - following. + following. There are also API calls for the follow features, see the Action + API reference documentation. v1.7.1 2012-06-20 ================= From c8bc38c62e86a6c834c82fbad78ab233515ead43 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 28 Jun 2012 11:10:55 +0100 Subject: [PATCH 089/155] [#2389] Fix tests that were failing if 'error' was present on the JS strings --- ckan/tests/functional/test_group.py | 2 +- ckan/tests/functional/test_package.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/tests/functional/test_group.py b/ckan/tests/functional/test_group.py index e71cd286b1e..8b89aa6c7d9 100644 --- a/ckan/tests/functional/test_group.py +++ b/ckan/tests/functional/test_group.py @@ -460,7 +460,7 @@ def test_1_do_diff(self): res = form.submit() res = res.follow() main_res = self.main_div(res) - assert 'error' not in main_res.lower(), main_res + assert 'form-errors' not in main_res.lower(), main_res assert 'Revision Differences' in main_res, main_res assert self.grp.name in main_res, main_res assert 'description
    - Written by Puccini\n+ Written off
    ' in main_res, main_res diff --git a/ckan/tests/functional/test_package.py b/ckan/tests/functional/test_package.py index 33d88f4f22c..ffe423737fb 100644 --- a/ckan/tests/functional/test_package.py +++ b/ckan/tests/functional/test_package.py @@ -1446,7 +1446,7 @@ def test_1_do_diff(self): res = form.submit() res = res.follow() main_res = self.main_div(res) - assert 'error' not in main_res.lower(), main_res + assert 'form-errors' not in main_res.lower(), main_res assert 'Revision Differences' in main_res, main_res assert self.pkg1.name in main_res, main_res assert 'notes
    - Written by Puccini\n+ Written off
    ' in main_res, main_res From d290e736397f26be37191f3fadedebe8690804b1 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 28 Jun 2012 12:46:57 +0200 Subject: [PATCH 090/155] [#2592] Add new source install instructions to CHANGELOG --- CHANGELOG.txt | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index adaca8a3cf4..30175d6a057 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -4,16 +4,19 @@ CKAN CHANGELOG v1.8 ==== -* requirements have been updated see doc/install-from-source.rst -users will need to do a new pip install (#2592) -* New 'follow' feature. You'll now see a 'Followers' tab on user and dataset - pages, where you can see how many users are following that user or dataset. - If you're logged in, you'll see a 'Follow' button on the pages of datasets - and other users that you can click to follow them. Also when logged in, if - you go to your own user page you'll see a new 'Dashboard' tab where you can - see an activity stream from of all the users and datasets that you're - following. There are also API calls for the follow features, see the Action - API reference documentation. +* [#2592,#2428] Ubuntu 12.04 Precise is now supported with CKAN source install. + The source install instructions have been updated and simplified. + Some of CKAN's dependencies have been updated and some removed. +* Requirements have been updated see doc/install-from-source.rst + users will need to do a new pip install (#2592) +* [#2304,#2305] New 'follow' feature. You'll now see a 'Followers' tab on user + and dataset pages, where you can see how many users are following that user + or dataset. If you're logged in, you'll see a 'Follow' button on the pages + of datasets and other users that you can click to follow them. Also when + logged in, if you go to your own user page you'll see a new 'Dashboard' tab + where you can see an activity stream from of all the users and datasets that + you're following. There are also API calls for the follow features, see the + Action API reference documentation. v1.7.1 2012-06-20 ================= From 890e95ffa81a30e657be1ae693824725b5a942ba Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Thu, 28 Jun 2012 11:38:14 +0100 Subject: [PATCH 091/155] Indents the related tab when looking at a list of related items --- ckan/controllers/related.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/controllers/related.py b/ckan/controllers/related.py index f6f425deec5..e90b4be4938 100644 --- a/ckan/controllers/related.py +++ b/ckan/controllers/related.py @@ -34,5 +34,5 @@ def list(self, id): base.abort(401, base._('Unauthorized to read package %s') % id) c.related_count = len(c.pkg.related) - + c.action = 'related' return base.render("package/related_list.html") From fcc52c04adc14276e99e66cb5adb63674f8d3fbe Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 28 Jun 2012 12:22:30 +0100 Subject: [PATCH 092/155] Remove unnecessary exception params that were causing tests to fail due to Unicode issues --- ckan/controllers/package.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index ef086fc5ae0..ccab6931135 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -594,8 +594,7 @@ def _save_new(self, context, package_type=None): except DataError: abort(400, _(u'Integrity Error')) except SearchIndexError, e: - abort(500, _(u'Unable to add package to search index.') + - repr(e.args)) + abort(500, _(u'Unable to add package to search index.')) except ValidationError, e: errors = e.error_dict error_summary = e.error_summary @@ -627,7 +626,7 @@ def _save_edit(self, name_or_id, context): except DataError: abort(400, _(u'Integrity Error')) except SearchIndexError, e: - abort(500, _(u'Unable to update search index.') + repr(e.args)) + abort(500, _(u'Unable to update search index.')) except ValidationError, e: errors = e.error_dict error_summary = e.error_summary From 9ac8976d20a785abafad6f78d40f2699e6aa31a9 Mon Sep 17 00:00:00 2001 From: Toby Date: Thu, 28 Jun 2012 13:34:41 +0100 Subject: [PATCH 093/155] [#2289] include discovered locales --- ckan/lib/i18n.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckan/lib/i18n.py b/ckan/lib/i18n.py index 5b7ed0dad0f..94335ba180a 100644 --- a/ckan/lib/i18n.py +++ b/ckan/lib/i18n.py @@ -17,7 +17,8 @@ def get_locales_from_config(): filtered_out = config.get('ckan.locales_filtered_out', '').split() locale_default = config.get('ckan.locale_default', 'en') locale_order = config.get('ckan.locale_order', '').split() - all_locales = set(locales_offered) | set(locale_order) | set(locale_default) + known_locales = get_locales() + all_locales = set(known_locales) | set(locales_offered) | set(locale_order) | set(locale_default) all_locales -= set(filtered_out) return all_locales From f50cd42e3065dde811b133b9266e444a9b1b9a2c Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 28 Jun 2012 15:36:52 +0200 Subject: [PATCH 094/155] [#2615] Add activities when following something Add activity stream activities when a user starts to follow a dataset or another user. --- ckan/logic/action/create.py | 36 ++++++ ckan/logic/action/get.py | 17 ++- ckan/logic/validators.py | 2 + .../activity_streams/follow_dataset.html | 14 +++ .../activity_streams/follow_user.html | 14 +++ ckan/tests/functional/api/test_activity.py | 107 +++++++++++++++++- ckan/tests/functional/test_activity.py | 15 +++ 7 files changed, 201 insertions(+), 4 deletions(-) create mode 100644 ckan/templates/activity_streams/follow_dataset.html create mode 100644 ckan/templates/activity_streams/follow_user.html diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 874abc88685..bf732dfa08a 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -842,6 +842,7 @@ def follow_user(context, data_dict): raise logic.NotAuthorized model = context['model'] + session = context['session'] userobj = model.User.get(context['user']) if not userobj: @@ -869,6 +870,23 @@ def follow_user(context, data_dict): follower = model_save.user_following_user_dict_save(data_dict, context) + activity_dict = { + 'user_id': userobj.id, + 'object_id': data_dict['id'], + 'activity_type': 'follow user', + } + activity_dict['data'] = { + 'user': ckan.lib.dictization.table_dictize( + model.User.get(data_dict['id']), context), + } + activity_create_context = { + 'model': model, + 'user': userobj, + 'defer_commit':True, + 'session': session + } + activity_create(activity_create_context, activity_dict, ignore_auth=True) + if not context.get('defer_commit'): model.repo.commit() @@ -895,6 +913,7 @@ def follow_dataset(context, data_dict): raise logic.NotAuthorized model = context['model'] + session = context['session'] userobj = model.User.get(context['user']) if not userobj: @@ -918,6 +937,23 @@ def follow_dataset(context, data_dict): follower = model_save.user_following_dataset_dict_save(data_dict, context) + activity_dict = { + 'user_id': userobj.id, + 'object_id': data_dict['id'], + 'activity_type': 'follow dataset', + } + activity_dict['data'] = { + 'dataset': ckan.lib.dictization.table_dictize( + model.Package.get(data_dict['id']), context), + } + activity_create_context = { + 'model': model, + 'user': userobj, + 'defer_commit':True, + 'session': session + } + activity_create(activity_create_context, activity_dict, ignore_auth=True) + if not context.get('defer_commit'): model.repo.commit() diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index a8a6fa9c506..f15525bd768 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1746,6 +1746,14 @@ def _render_deleted_group_activity(context, activity): return _render('activity_streams/deleted_group.html', extra_vars = {'activity': activity}) +def _render_follow_dataset_activity(context, activity): + return _render('activity_streams/follow_dataset.html', + extra_vars = {'activity': activity}) + +def _render_follow_user_activity(context, activity): + return _render('activity_streams/follow_user.html', + extra_vars = {'activity': activity}) + # Global dictionary mapping activity types to functions that render activity # dicts to HTML snippets for including in HTML pages. activity_renderers = { @@ -1757,6 +1765,8 @@ def _render_deleted_group_activity(context, activity): 'new group' : _render_new_group_activity, 'changed group' : _render_changed_group_activity, 'deleted group' : _render_deleted_group_activity, + 'follow dataset': _render_follow_dataset_activity, + 'follow user': _render_follow_user_activity, } def _activity_list_to_html(context, activity_stream): @@ -2057,15 +2067,16 @@ def dashboard_activity_list(context, data_dict): user_id = _get_or_bust(data_dict, 'id') activity_query = model.Session.query(model.Activity) - user_query = activity_query; user_followees_query = activity_query.join(model.UserFollowingUser, model.UserFollowingUser.object_id == model.Activity.user_id) dataset_followees_query = activity_query.join(model.UserFollowingDataset, model.UserFollowingDataset.object_id == model.Activity.object_id) - user_query = user_query.filter(model.Activity.user_id==user_id) + from_user_query = activity_query.filter(model.Activity.user_id==user_id) + about_user_query = activity_query.filter(model.Activity.object_id==user_id) user_followees_query = user_followees_query.filter(model.UserFollowingUser.follower_id==user_id) dataset_followees_query = dataset_followees_query.filter(model.UserFollowingDataset.follower_id==user_id) - query = user_query.union(user_followees_query).union(dataset_followees_query) + query = from_user_query.union(about_user_query).union( + user_followees_query).union(dataset_followees_query) query = query.order_by(_desc(model.Activity.timestamp)) query = query.limit(15) activity_objects = query.all() diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index 30f4fea47f3..d7bc95c15d6 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -152,8 +152,10 @@ def activity_type_exists(activity_type): 'new package' : package_id_exists, 'changed package' : package_id_exists, 'deleted package' : package_id_exists, + 'follow dataset' : package_id_exists, 'new user' : user_id_exists, 'changed user' : user_id_exists, + 'follow user' : user_id_exists, 'new group' : group_id_exists, 'changed group' : group_id_exists, 'deleted group' : group_id_exists, diff --git a/ckan/templates/activity_streams/follow_dataset.html b/ckan/templates/activity_streams/follow_dataset.html new file mode 100644 index 00000000000..5a0c8f7f32e --- /dev/null +++ b/ckan/templates/activity_streams/follow_dataset.html @@ -0,0 +1,14 @@ + +${h.activity_div( + template=_("{actor} started following {object}"), + activity=activity, + actor=h.linked_user(activity.user_id), + object=h.dataset_link(activity.data.dataset), + )} + diff --git a/ckan/templates/activity_streams/follow_user.html b/ckan/templates/activity_streams/follow_user.html new file mode 100644 index 00000000000..f0e22d1f64b --- /dev/null +++ b/ckan/templates/activity_streams/follow_user.html @@ -0,0 +1,14 @@ + +${h.activity_div( + template=_("{actor} started following {object}"), + activity=activity, + actor=h.linked_user(activity.user_id), + object=h.linked_user(activity.data.user.name), + )} + diff --git a/ckan/tests/functional/api/test_activity.py b/ckan/tests/functional/api/test_activity.py index 4814b064d0e..880efb9e4ca 100644 --- a/ckan/tests/functional/api/test_activity.py +++ b/ckan/tests/functional/api/test_activity.py @@ -4,7 +4,9 @@ import ckan import ckan.model as model -from ckan.logic.action.create import package_create as _package_create, user_create, group_create +from ckan.logic.action.create import package_create as _package_create +from ckan.logic.action.create import user_create, group_create, follow_dataset +from ckan.logic.action.create import follow_user from ckan.logic.action.update import package_update as _package_update, resource_update from ckan.logic.action.update import user_update, group_update from ckan.logic.action.delete import package_delete @@ -2025,3 +2027,106 @@ def test_delete_extras_not_logged_in(self): "Need some packages with extras to test") for package_dict in packages_with_extras: self._delete_extra(package_dict, None) + + def test_follow_dataset(self): + user = self.normal_user + before = self.record_details(user.id) + context = { + 'model': model, + 'session': model.Session, + 'user': user.name, + } + data = {'id': self.warandpeace.id} + follow_dataset(context, data) + after = self.record_details(user.id, self.warandpeace.id) + + # Find the new activity in the user's activity stream. + user_new_activities = (find_new_activities( + before['user activity stream'], after['user activity stream'])) + assert len(user_new_activities) == 1, ("There should be 1 new " + " activity in the user's activity stream, but found %i" % + len(user_new_activities)) + activity = user_new_activities[0] + + # The same new activity should appear in the package's activity stream. + pkg_new_activities = after['package activity stream'] + for activity in user_new_activities: + assert activity in pkg_new_activities + + self.check_dashboard(before, after, user_new_activities, [user.id]) + + # Check that the new activity has the right attributes. + assert activity['object_id'] == self.warandpeace.id, \ + str(activity['object_id']) + assert activity['user_id'] == user.id, str(activity['user_id']) + assert activity['activity_type'] == 'follow dataset', \ + str(activity['activity_type']) + if not activity.has_key('id'): + assert False, "activity object should have an id value" + # TODO: Test for the _correct_ revision_id value. + if not activity.has_key('revision_id'): + assert False, "activity object should have a revision_id value" + timestamp = datetime_from_string(activity['timestamp']) + assert timestamp >= before['time'] and timestamp <= \ + after['time'], str(activity['timestamp']) + + assert len(self.activity_details(activity)) == 0 + + def test_follow_user(self): + user = self.normal_user + before = self.record_details(user.id) + followee_before = self.record_details(self.sysadmin_user.id) + context = { + 'model': model, + 'session': model.Session, + 'user': user.name, + } + data = {'id': self.sysadmin_user.id} + follow_user(context, data) + after = self.record_details(user.id) + followee_after = self.record_details(self.sysadmin_user.id) + + # Find the new activity in the user's activity stream. + user_new_activities = (find_new_activities( + before['user activity stream'], after['user activity stream'])) + assert len(user_new_activities) == 1, ("There should be 1 new " + " activity in the user's activity stream, but found %i" % + len(user_new_activities)) + activity = user_new_activities[0] + + # Check that the new activity appears in the user's private activity + # stream. + user_new_activities = (find_new_activities( + before['follower dashboard activity stream'], + after['follower dashboard activity stream'])) + assert len(user_new_activities) == 1, ("There should be 1 new " + " activity in the user's activity stream, but found %i" % + len(user_new_activities)) + assert user_new_activities[0] == activity + + # Check that the new activity appears in the followee's private + # activity stream. + followee_new_activities = (find_new_activities( + followee_before['follower dashboard activity stream'], + followee_after['follower dashboard activity stream'])) + assert len(followee_new_activities) == 1, ("There should be 1 new " + " activity in the user's activity stream, but found %i" % + len(followee_new_activities)) + assert followee_new_activities[0] == activity + + # Check that the new activity has the right attributes. + assert activity['object_id'] == self.sysadmin_user.id, \ + str(activity['object_id']) + assert activity['user_id'] == user.id, str(activity['user_id']) + assert activity['activity_type'] == 'follow user', \ + str(activity['activity_type']) + if not activity.has_key('id'): + assert False, "activity object should have an id value" + # TODO: Test for the _correct_ revision_id value. + if not activity.has_key('revision_id'): + assert False, "activity object should have a revision_id value" + timestamp = datetime_from_string(activity['timestamp']) + assert timestamp >= before['time'] and timestamp <= \ + after['time'], str(activity['timestamp']) + + assert len(self.activity_details(activity)) == 0 diff --git a/ckan/tests/functional/test_activity.py b/ckan/tests/functional/test_activity.py index fccdfb42f93..56772ce3374 100644 --- a/ckan/tests/functional/test_activity.py +++ b/ckan/tests/functional/test_activity.py @@ -3,6 +3,7 @@ import ckan from routes import url_for from ckan.logic.action.create import package_create, user_create, group_create +from ckan.logic.action.create import follow_dataset, follow_user from ckan.logic.action.update import package_update, resource_update from ckan.logic.action.update import user_update, group_update from ckan.logic.action.delete import package_delete @@ -113,6 +114,20 @@ def test_user_activity(self): (user['fullname'], resource['name'], package['title']) \ in stripped, stripped + # Follow the package. + follow_dataset(context, {'id': package['id']}) + result = self.app.get(offset, status=200) + stripped = self.strip_tags(result) + assert '%s started following %s' % (user['fullname'], + package['title']) in stripped, stripped + + # Follow another user. + follow_user(context, {'id': 'joeadmin'}) + result = self.app.get(offset, status=200) + stripped = self.strip_tags(result) + assert '%s started following %s' % (user['fullname'], + 'joeadmin') in stripped, stripped + # Create a new group. group = { 'name': 'baseball-stats-group', From 52de660da3cd4b84b0ed62e9c7885b065ef76953 Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 25 Jun 2012 15:50:53 +0100 Subject: [PATCH 095/155] Fix stats extension frontend --- .../stats/templates/ckanext/stats/index.html | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/ckanext/stats/templates/ckanext/stats/index.html b/ckanext/stats/templates/ckanext/stats/index.html index 681527d330d..ec14a95efdf 100644 --- a/ckanext/stats/templates/ckanext/stats/index.html +++ b/ckanext/stats/templates/ckanext/stats/index.html @@ -19,8 +19,8 @@ width: 950px; } - .legend table { - width: auto; + h3 { + margin-top: 20px; } .graph { @@ -28,6 +28,11 @@ height: 300px; margin-bottom: 20px; } + + .metric { + width: 30%; + } + @@ -47,8 +52,8 @@

    Revisions to Datasets per week

    Top Rated Datasets

    - - +
    DatasetAverage ratingNumber of ratings
    + @@ -56,32 +61,32 @@

    Top Rated Datasets

    No ratings

    Most Edited Datasets

    -
    DatasetAverage ratingNumber of ratings
    ${h.link_to(package.title or package.name, h.url_for(controller='package', action='read', id=package.name))}${rating}${num_ratings}
    - +
    DatasetNumber of edits
    +
    DatasetNumber of edits
    ${h.link_to(package.title or package.name, h.url_for(controller='package', action='read', id=package.name))}${edits}

    Largest Groups

    - - +
    GroupNumber of datasets
    +
    GroupNumber of datasets
    ${h.link_to(group.title or group.name, h.url_for(controller='group', action='read', id=group.name))}${num_packages}

    Top Tags

    - +
    - +
    ${h.link_to(tag.name, h.url_for(controller='tag', action='read', id=tag.name))}${num_packages}${h.link_to(tag.name, h.url_for(controller='tag', action='read', id=tag.name))}${num_packages}

    Users owning most datasets

    - +
    - +
    ${h.linked_user(user)}${num_packages}${h.linked_user(user)}${num_packages}
    From fea640d621a5006a600a2b7a67d972010d31da95 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Tue, 26 Jun 2012 10:38:22 +0100 Subject: [PATCH 096/155] Do not load excanvas.js in IE 9 IE 9 natively supports the canvas element and loading this fallback will break Flot. --- ckanext/stats/templates/ckanext/stats/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/stats/templates/ckanext/stats/index.html b/ckanext/stats/templates/ckanext/stats/index.html index ec14a95efdf..7eb37d0e317 100644 --- a/ckanext/stats/templates/ckanext/stats/index.html +++ b/ckanext/stats/templates/ckanext/stats/index.html @@ -105,7 +105,7 @@

    Users owning most datasets

    $('body').addClass('no-sidebar'); - + - - + ${jsConditionalForIe(8, '<script type="text/javascript" src="' + h.url_for_static('/scripts/vendor/flot/0.7/excanvas.js') + '"></script>', 'lte')} + - ${jsConditionalForIe(8, '<script type="text/javascript" src="' + h.url_for_static('/scripts/vendor/flot/0.7/excanvas.js') + '"></script>', 'lte')} + ${jsConditionalForIe(8, '<script language="javascript" type="text/javascript" src="' + h.url_for_static('/scripts/vendor/flot/0.7/excanvas.js') + '"></script>', 'lte')} From b9b9b7859c6cb9162da6430e9b49ecd22610a86f Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Tue, 26 Jun 2012 12:21:25 +0100 Subject: [PATCH 102/155] Add link to excanvas thread on the net --- ckanext/stats/templates/ckanext/stats/index.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ckanext/stats/templates/ckanext/stats/index.html b/ckanext/stats/templates/ckanext/stats/index.html index ab9dc4ec47c..e0165730c8d 100644 --- a/ckanext/stats/templates/ckanext/stats/index.html +++ b/ckanext/stats/templates/ckanext/stats/index.html @@ -10,8 +10,11 @@ - + From 23f57d93549e45277584996ac6eb1680401c0e0c Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Tue, 26 Jun 2012 12:40:38 +0100 Subject: [PATCH 103/155] Fix trailing commas in application.js This prevents exceptions being thrown in IE --- ckan/public/scripts/application.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/public/scripts/application.js b/ckan/public/scripts/application.js index 2eecaa34bd8..e2a36c76961 100644 --- a/ckan/public/scripts/application.js +++ b/ckan/public/scripts/application.js @@ -1443,7 +1443,7 @@ CKAN.Utils = function($, my) { return; } var data = JSON.stringify({ - id: object_id, + id: object_id }); var nextState = 'unfollow'; var nextString = CKAN.Strings.unfollow; @@ -1457,7 +1457,7 @@ CKAN.Utils = function($, my) { return; } var data = JSON.stringify({ - id: object_id, + id: object_id }); var nextState = 'follow'; var nextString = CKAN.Strings.follow; @@ -1476,7 +1476,7 @@ CKAN.Utils = function($, my) { success: function(data) { button.setAttribute('data-state', nextState); button.innerHTML = nextString; - }, + } }); }; From 0588164a70c4b406b41f7808652bedf06514753b Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 28 Jun 2012 13:05:45 +0100 Subject: [PATCH 104/155] Move logic from template to controller --- ckanext/stats/controller.py | 15 ++++++++ .../stats/templates/ckanext/stats/index.html | 35 +++---------------- 2 files changed, 20 insertions(+), 30 deletions(-) diff --git a/ckanext/stats/controller.py b/ckanext/stats/controller.py index 4e86ab1d98d..37114037b0a 100644 --- a/ckanext/stats/controller.py +++ b/ckanext/stats/controller.py @@ -17,6 +17,21 @@ def index(self): c.deleted_packages_by_week = rev_stats.get_by_week('deleted_packages') c.num_packages_by_week = rev_stats.get_num_packages_by_week() c.package_revisions_by_week = rev_stats.get_by_week('package_revisions') + + c.packages_by_week = []; + for week_date, num_packages, cumulative_num_packages in c.num_packages_by_week: + c.packages_by_week.append('[new Date(%s), %s]' % (week_date.replace('-', ','), cumulative_num_packages)); + + + c.all_package_revisions = []; + for week_date, revs, num_revisions, cumulative_num_revisions in c.package_revisions_by_week: + c.all_package_revisions.append('[new Date(%s), %s]' % (week_date.replace('-', ','), num_revisions)); + + c.new_datasets = [] + for week_date, pkgs, num_packages, cumulative_num_packages in c.new_packages_by_week: + c.new_datasets.append('[new Date(%s), %s]' % (week_date.replace('-', ','), num_packages)); + + return p.toolkit.render('ckanext/stats/index.html') def leaderboard(self, id=None): diff --git a/ckanext/stats/templates/ckanext/stats/index.html b/ckanext/stats/templates/ckanext/stats/index.html index e0165730c8d..927ca1add18 100644 --- a/ckanext/stats/templates/ckanext/stats/index.html +++ b/ckanext/stats/templates/ckanext/stats/index.html @@ -129,17 +129,9 @@

    Users owning most datasets

    position: "nw" } }; - - var data = [ [ - ${",\n".join(packages_by_week)} + ${",".join(c.packages_by_week)} ] ]; $.plot($("#new_packages_graph"), data, options); @@ -156,30 +148,13 @@

    Users owning most datasets

    }, colors: ["#ffcc33", "#ff8844"] }; - var data = [ {label: "All package revisions", - lines: { - fill: 1 - }, - data: [ - ${",\n".join(all_package_revisions)} - ]}, + lines: {fill: 1 }, + data: [${",".join(c.all_package_revisions)}]}, {label: "New datasets", - lines: { - fill: 1 - }, - data: [ - ${",\n".join(new_datasets)} - ]} + lines: {fill: 1}, + data: [${",".join(c.new_datasets)}]} ]; $.plot($("#package_revisions_graph"), data, options); From 53baba208e350a0619300a79d755026e7ce6dea1 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 28 Jun 2012 16:45:31 +0100 Subject: [PATCH 105/155] [#1732] Update schema CHANGELOG (no new version) --- ckan/config/solr/CHANGELOG.txt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ckan/config/solr/CHANGELOG.txt b/ckan/config/solr/CHANGELOG.txt index 735d6a8dc6c..eb600e042cc 100644 --- a/ckan/config/solr/CHANGELOG.txt +++ b/ckan/config/solr/CHANGELOG.txt @@ -1,11 +1,6 @@ CKAN SOLR schemas changelog =========================== -v1.4 - (ckan>=1.8) ------------------- -* Add vocab_* dynamic field so it is possible to facet by vocabulary tags -* Add copyField for text with source vocab_* - v1.4 - (ckan>=1.7) -------------------- * Add Ascii folding filter to text fields. @@ -13,6 +8,8 @@ v1.4 - (ckan>=1.7) * Add title_string so you can sort alphabetically on title. * Fields related to analytics, access and view counts. * Add data_dict field for the whole package_dict. +* Add vocab_* dynamic field so it is possible to facet by vocabulary tags +* Add copyField for text with source vocab_* v1.3 - (ckan>=1.5.1) -------------------- From 5dd74fb1f9a1ffda82d89d1d53f24f69c25bb134 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 28 Jun 2012 18:15:24 +0200 Subject: [PATCH 106/155] [#2615] Use get_action() to get activity_create --- ckan/logic/action/create.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index bf732dfa08a..ce44abfa249 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -534,7 +534,8 @@ def group_create(context, data_dict): 'defer_commit':True, 'session': session } - activity_create(activity_create_context, activity_dict, ignore_auth=True) + logic.get_action('activity_create')(activity_create_context, + activity_dict, ignore_auth=True) if not context.get('defer_commit'): model.repo.commit() @@ -648,7 +649,8 @@ def user_create(context, data_dict): 'object_id': user.id, 'activity_type': 'new user', } - activity_create(activity_create_context, activity_dict, ignore_auth=True) + logic.get_action('activity_create')(activity_create_context, + activity_dict, ignore_auth=True) if not context.get('defer_commit'): model.repo.commit() @@ -885,7 +887,8 @@ def follow_user(context, data_dict): 'defer_commit':True, 'session': session } - activity_create(activity_create_context, activity_dict, ignore_auth=True) + logic.get_action('activity_create')(activity_create_context, + activity_dict, ignore_auth=True) if not context.get('defer_commit'): model.repo.commit() @@ -952,7 +955,8 @@ def follow_dataset(context, data_dict): 'defer_commit':True, 'session': session } - activity_create(activity_create_context, activity_dict, ignore_auth=True) + logic.get_action('activity_create')(activity_create_context, + activity_dict, ignore_auth=True) if not context.get('defer_commit'): model.repo.commit() From c9e0e044494d0500bfdd88023e0a03b8a7012125 Mon Sep 17 00:00:00 2001 From: Toby Date: Thu, 28 Jun 2012 17:27:56 +0100 Subject: [PATCH 107/155] [#2289] add some docstrings etc to lib/i18n --- ckan/lib/i18n.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ckan/lib/i18n.py b/ckan/lib/i18n.py index 94335ba180a..ca6c1682418 100644 --- a/ckan/lib/i18n.py +++ b/ckan/lib/i18n.py @@ -13,6 +13,8 @@ # translation currently. def get_locales_from_config(): + ''' despite the name of this function it gets the locales defined by + the config AND also the locals available subject to the config. ''' locales_offered = config.get('ckan.locales_offered', '').split() filtered_out = config.get('ckan.locales_filtered_out', '').split() locale_default = config.get('ckan.locale_default', 'en') @@ -23,6 +25,7 @@ def get_locales_from_config(): return all_locales def _get_locales(): + # FIXME this wants cleaning up and merging with get_locales_from_config() assert not config.get('lang'), \ '"lang" config option not supported - please use ckan.locale_default instead.' locales_offered = config.get('ckan.locales_offered', '').split() @@ -79,6 +82,8 @@ def get_locales(): return locales def non_translated_locals(): + ''' These are the locales that are available but for which there are + no translations. returns a list like ['en', 'de', ...] ''' global _non_translated_locals if not _non_translated_locals: locales = config.get('ckan.locale_order', '').split() From 646e551881c54c54b120f68f36205bb7207b53fc Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Thu, 28 Jun 2012 15:16:24 +0100 Subject: [PATCH 108/155] [#2439] Allow query parameter to be a list of strings As well as a single string. --- ckan/logic/action/get.py | 8 ++++---- ckan/tests/lib/test_tag_search.py | 5 +++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index a8a6fa9c506..f954b73af7d 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1245,10 +1245,10 @@ def resource_search(context, data_dict): def _tag_search(context, data_dict): model = context['model'] - query = data_dict.get('query') or data_dict.get('q') - if query: - query = query.strip() - terms = [query] if query else [] + terms = data_dict.get('query') or data_dict.get('q') or [] + if isinstance(terms, basestring): + terms = [terms] + terms = [ t.strip() for t in terms if t.strip() ] fields = data_dict.get('fields', {}) offset = data_dict.get('offset') diff --git a/ckan/tests/lib/test_tag_search.py b/ckan/tests/lib/test_tag_search.py index 9047e13e5aa..d6f4177675e 100644 --- a/ckan/tests/lib/test_tag_search.py +++ b/ckan/tests/lib/test_tag_search.py @@ -26,6 +26,11 @@ def test_good_search_query(self): assert 'russian' in result['results'], result assert 'tolstoy' in result['results'], result + def test_good_search_queries(self): + result = search.query_for(model.Tag).run(query=[u'ru', u's']) + assert result['count'] == 1, result + assert 'russian' in result['results'], result + def test_bad_search_query(self): result = search.query_for(model.Tag).run(query=u'asdf') assert result['count'] == 0, result From 903bd62d06392eb211925b4f63d2ee89570bbb78 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Thu, 28 Jun 2012 15:27:23 +0100 Subject: [PATCH 109/155] [#2439] Mark fields parameter as deprecated ... in tag_search and tag_autocomplete. --- ckan/lib/search/query.py | 9 ++++++++- ckan/logic/action/get.py | 12 ++++++++---- ckan/tests/logic/test_tag.py | 10 ++++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/ckan/lib/search/query.py b/ckan/lib/search/query.py index a3b7adf35fe..feef9622ec1 100644 --- a/ckan/lib/search/query.py +++ b/ckan/lib/search/query.py @@ -156,10 +156,17 @@ def run(self, query=[], fields={}, options=None, **kwargs): else: options.update(kwargs) + if isinstance(query, basestring): + query = [query] + + query = query[:] + for field, value in fields.items(): + if field in ('tag', 'tags'): + query.append(value) + context = {'model': model, 'session': model.Session} data_dict = { 'query': query, - 'fields': fields, 'offset': options.get('offset'), 'limit': options.get('limit') } diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index f954b73af7d..fce9a0c3392 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1250,6 +1250,10 @@ def _tag_search(context, data_dict): terms = [terms] terms = [ t.strip() for t in terms if t.strip() ] + if 'fields' in data_dict: + log.warning('"fields" parameter is deprecated. ' + 'Use the "query" parameter instead') + fields = data_dict.get('fields', {}) offset = data_dict.get('offset') limit = data_dict.get('limit') @@ -1293,12 +1297,12 @@ def tag_search(context, data_dict): searched. If the ``vocabulary_id`` argument is given then only tags belonging to that vocabulary will be searched instead. - :param query: the string to search for - :type query: string + :param query: the string(s) to search for + :type query: string or list of strings :param vocabulary_id: the id or name of the tag vocabulary to search in (optional) :type vocabulary_id: string - :param fields: + :param fields: deprecated :type fields: dictionary :param limit: the maximum number of tags to return :type limit: int @@ -1334,7 +1338,7 @@ def tag_autocomplete(context, data_dict): :param vocabulary_id: the id or name of the tag vocabulary to search in (optional) :type vocabulary_id: string - :param fields: + :param fields: deprecated :type fields: dictionary :param limit: the maximum number of tags to return :type limit: int diff --git a/ckan/tests/logic/test_tag.py b/ckan/tests/logic/test_tag.py index 2e96835abad..1f710ab6563 100644 --- a/ckan/tests/logic/test_tag.py +++ b/ckan/tests/logic/test_tag.py @@ -202,6 +202,16 @@ def test_15a_tag_search_with_one_match(self): assert len(tag_dicts) == 1 assert tag_dicts[0]['name'] == 'russian' + def test_15a_tag_search_with_one_match_using_fields_parameter(self): + paramd = {'fields': {'tags': 'russ'} } + params = json.dumps(paramd) + res = self.app.post('/api/action/tag_search', params=params) + assert res.json['success'] is True + assert res.json['result']['count'] == 1 + tag_dicts = res.json['result']['results'] + assert len(tag_dicts) == 1 + assert tag_dicts[0]['name'] == 'russian' + def test_15a_tag_search_with_many_matches(self): paramd = {'q': 'tol' } params = json.dumps(paramd) From a719c132afe400ea038a4340c09d46c1a34ea068 Mon Sep 17 00:00:00 2001 From: Toby Date: Thu, 28 Jun 2012 18:02:25 +0100 Subject: [PATCH 110/155] [xs] whitespace fixes of test --- ckan/tests/functional/api/test_activity.py | 72 +++++++++++----------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/ckan/tests/functional/api/test_activity.py b/ckan/tests/functional/api/test_activity.py index 880efb9e4ca..4662909ca88 100644 --- a/ckan/tests/functional/api/test_activity.py +++ b/ckan/tests/functional/api/test_activity.py @@ -229,7 +229,7 @@ def _create_package(self, user, name=None): user_new_activities = (find_new_activities( before['user activity stream'], after['user activity stream'])) assert len(user_new_activities) == 1, ("There should be 1 new " - " activity in the user's activity stream, but found %i" % + " activity in the user's activity stream, but found %i" % len(user_new_activities)) activity = user_new_activities[0] @@ -279,20 +279,20 @@ def _create_package(self, user, name=None): str(detail['activity_id']) if detail['object_id'] == package_created['id']: - assert detail['activity_type'] == "new", ( + assert detail['activity_type'] == "new", ( str(detail['activity_type'])) assert detail['object_type'] == "Package", \ str(detail['object_type']) elif (detail['object_id'] in [resource['id'] for resource in package_created['resources']]): - assert detail['activity_type'] == "new", ( + assert detail['activity_type'] == "new", ( str(detail['activity_type'])) assert detail['object_type'] == "Resource", ( str(detail['object_type'])) else: - assert detail['activity_type'] == "added", ( + assert detail['activity_type'] == "added", ( str(detail['activity_type'])) assert detail['object_type'] == "tag", ( str(detail['object_type'])) @@ -337,7 +337,7 @@ def _add_resource(self, package, user): user_new_activities = (find_new_activities( before['user activity stream'], after['user activity stream'])) assert len(user_new_activities) == 1, ("There should be 1 new " - " activity in the user's activity stream, but found %i" % + " activity in the user's activity stream, but found %i" % len(user_new_activities)) activity = user_new_activities[0] @@ -381,9 +381,9 @@ def _add_resource(self, package, user): resource_ids_before] assert len(new_resource_ids) == 1 new_resource_id = new_resource_ids[0] - assert detail['object_id'] == new_resource_id, ( + assert detail['object_id'] == new_resource_id, ( str(detail['object_id'])) - assert detail['object_type'] == "Resource", ( + assert detail['object_type'] == "Resource", ( str(detail['object_type'])) assert detail['activity_type'] == "new", ( str(detail['activity_type'])) @@ -428,7 +428,7 @@ def _delete_extra(self, package_dict, user): user_new_activities = (find_new_activities( before['user activity stream'], after['user activity stream'])) assert len(user_new_activities) == 1, ("There should be 1 new " - " activity in the user's activity stream, but found %i" % + " activity in the user's activity stream, but found %i" % len(user_new_activities)) activity = user_new_activities[0] @@ -524,7 +524,7 @@ def _update_extra(self, package_dict, user): user_new_activities = (find_new_activities( before['user activity stream'], after['user activity stream'])) assert len(user_new_activities) == 1, ("There should be 1 new " - " activity in the user's activity stream, but found %i" % + " activity in the user's activity stream, but found %i" % len(user_new_activities)) activity = user_new_activities[0] @@ -616,7 +616,7 @@ def _add_extra(self, package_dict, user, key=None): user_new_activities = (find_new_activities( before['user activity stream'], after['user activity stream'])) assert len(user_new_activities) == 1, ("There should be 1 new " - " activity in the user's activity stream, but found %i" % + " activity in the user's activity stream, but found %i" % len(user_new_activities)) activity = user_new_activities[0] @@ -672,7 +672,7 @@ def _add_extra(self, package_dict, user, key=None): def _create_activity(self, user, package, params): before = self.record_details(user.id, package.id) - response = self.app.post('/api/action/activity_create', + response = self.app.post('/api/action/activity_create', params=json.dumps(params), extra_environ={'Authorization': str(self.sysadmin_user.apikey)}) assert response.json['success'] == True @@ -683,7 +683,7 @@ def _create_activity(self, user, package, params): user_new_activities = (find_new_activities( before['user activity stream'], after['user activity stream'])) assert len(user_new_activities) == 1, ("There should be 1 new " - " activity in the user's activity stream, but found %i" % + " activity in the user's activity stream, but found %i" % len(user_new_activities)) activity = user_new_activities[0] @@ -879,7 +879,7 @@ def _delete_resources(self, package): user_new_activities = (find_new_activities( before['user activity stream'], after['user activity stream'])) assert len(user_new_activities) == 1, ("There should be 1 new " - " activity in the user's activity stream, but found %i" % + " activity in the user's activity stream, but found %i" % len(user_new_activities)) activity = user_new_activities[0] @@ -964,7 +964,7 @@ def _update_package(self, package, user): user_new_activities = (find_new_activities( before['user activity stream'], after['user activity stream'])) assert len(user_new_activities) == 1, ("There should be 1 new " - " activity in the user's activity stream, but found %i" % + " activity in the user's activity stream, but found %i" % len(user_new_activities)) activity = user_new_activities[0] @@ -1005,7 +1005,7 @@ def _update_package(self, package, user): assert detail['activity_id'] == activity['id'], \ str(detail['activity_id']) assert detail['object_id'] == package.id, str(detail['object_id']) - assert detail['object_type'] == "Package", ( + assert detail['object_type'] == "Package", ( str(detail['object_type'])) assert detail['activity_type'] == "changed", ( str(detail['activity_type'])) @@ -1042,7 +1042,7 @@ def _update_resource(self, package, resource, user): user_new_activities = (find_new_activities( before['user activity stream'], after['user activity stream'])) assert len(user_new_activities) == 1, ("There should be 1 new " - " activity in the user's activity stream, but found %i" % + " activity in the user's activity stream, but found %i" % len(user_new_activities)) activity = user_new_activities[0] @@ -1112,7 +1112,7 @@ def _delete_package(self, package): user_new_activities = (find_new_activities( before['user activity stream'], after['user activity stream'])) assert len(user_new_activities) == 1, ("There should be 1 new " - " activity in the user's activity stream, but found %i" % + " activity in the user's activity stream, but found %i" % len(user_new_activities)) activity = user_new_activities[0] @@ -1150,7 +1150,7 @@ def _delete_package(self, package): # Test for the presence of a correct activity detail item. details = self.activity_details(activity) assert len(details) == 1 - detail = details[0] + detail = details[0] assert detail['activity_id'] == activity['id'], \ str(detail['activity_id']) assert detail['object_id'] == package.id, str(detail['object_id']) @@ -1225,7 +1225,7 @@ def test_01_remove_tag(self): user_new_activities = (find_new_activities( before['user activity stream'], after['user activity stream'])) assert len(user_new_activities) == 1, ("There should be 1 new " - " activity in the user's activity stream, but found %i" % + " activity in the user's activity stream, but found %i" % len(user_new_activities)) activity = user_new_activities[0] @@ -1565,7 +1565,7 @@ def test_add_tag(self): user_new_activities = (find_new_activities( before['user activity stream'], after['user activity stream'])) assert len(user_new_activities) == 1, ("There should be 1 new " - " activity in the user's activity stream, but found %i" % + " activity in the user's activity stream, but found %i" % len(user_new_activities)) activity = user_new_activities[0] @@ -1644,7 +1644,7 @@ def test_activity_create_no_authorization(self): 'object_id': self.warandpeace.id, 'activity_type': 'changed package', } - response = self.app.post('/api/action/activity_create', + response = self.app.post('/api/action/activity_create', params=json.dumps(params), status=403) assert response.json['success'] == False @@ -1659,7 +1659,7 @@ def test_activity_create_not_authorized(self): 'object_id': self.warandpeace.id, 'activity_type': 'changed package', } - response = self.app.post('/api/action/activity_create', + response = self.app.post('/api/action/activity_create', params=json.dumps(params), extra_environ={'Authorization': str(self.normal_user.apikey)}, status=403) @@ -1676,7 +1676,7 @@ def test_activity_create_authorization_not_exists(self): 'object_id': self.warandpeace.id, 'activity_type': 'changed package', } - response = self.app.post('/api/action/activity_create', + response = self.app.post('/api/action/activity_create', params=json.dumps(params), extra_environ={'Authorization': 'xxxxxxxxxx'}, status=403) @@ -1697,9 +1697,9 @@ def test_activity_create_with_id(self): 'activity_type': 'changed package', } self._create_activity(self.sysadmin_user, self.warandpeace, params) - assert activity_id not in [activity['id'] for activity in + assert activity_id not in [activity['id'] for activity in self.user_activity_stream(user.id)] - assert activity_id not in [activity['id'] for activity in + assert activity_id not in [activity['id'] for activity in self.package_activity_stream(package.id)] def test_activity_create_with_timestamp(self): @@ -1714,7 +1714,7 @@ def test_activity_create_with_timestamp(self): 'timestamp': str(datetime.datetime.max), } self._create_activity(self.sysadmin_user, self.warandpeace, params) - params['timestamp'] = 'foobar' + params['timestamp'] = 'foobar' self._create_activity(self.sysadmin_user, self.warandpeace, params) def test_activity_create_with_revision(self): @@ -1732,9 +1732,9 @@ def test_activity_create_with_revision(self): 'activity_type': 'changed package', } self._create_activity(self.sysadmin_user, self.warandpeace, params) - assert revision_id not in [activity['revision_id'] for activity in + assert revision_id not in [activity['revision_id'] for activity in self.user_activity_stream(user.id)] - assert revision_id not in [activity['revision_id'] for activity in + assert revision_id not in [activity['revision_id'] for activity in self.package_activity_stream(package.id)] def test_activity_create_user_id_missing(self): @@ -1746,7 +1746,7 @@ def test_activity_create_user_id_missing(self): 'object_id': self.warandpeace.id, 'activity_type': 'changed package', } - response = self.app.post('/api/action/activity_create', + response = self.app.post('/api/action/activity_create', params=json.dumps(params), extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, status=409) @@ -1765,7 +1765,7 @@ def test_activity_create_user_id_empty(self): 'object_id': self.warandpeace.id, 'activity_type': 'changed package', } - response = self.app.post('/api/action/activity_create', + response = self.app.post('/api/action/activity_create', params=json.dumps(params), extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, status=409) @@ -1775,7 +1775,7 @@ def test_activity_create_user_id_empty(self): response.json['error'][u'user_id']) params['user_id'] = None - response = self.app.post('/api/action/activity_create', + response = self.app.post('/api/action/activity_create', params=json.dumps(params), extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, status=409) @@ -1794,7 +1794,7 @@ def test_activity_create_user_id_does_not_exist(self): 'object_id': self.warandpeace.id, 'activity_type': 'changed package', } - response = self.app.post('/api/action/activity_create', + response = self.app.post('/api/action/activity_create', params=json.dumps(params), extra_environ={'Authorization': str(self.sysadmin_user.apikey)}, status=409) @@ -2044,7 +2044,7 @@ def test_follow_dataset(self): user_new_activities = (find_new_activities( before['user activity stream'], after['user activity stream'])) assert len(user_new_activities) == 1, ("There should be 1 new " - " activity in the user's activity stream, but found %i" % + " activity in the user's activity stream, but found %i" % len(user_new_activities)) activity = user_new_activities[0] @@ -2090,7 +2090,7 @@ def test_follow_user(self): user_new_activities = (find_new_activities( before['user activity stream'], after['user activity stream'])) assert len(user_new_activities) == 1, ("There should be 1 new " - " activity in the user's activity stream, but found %i" % + " activity in the user's activity stream, but found %i" % len(user_new_activities)) activity = user_new_activities[0] @@ -2100,7 +2100,7 @@ def test_follow_user(self): before['follower dashboard activity stream'], after['follower dashboard activity stream'])) assert len(user_new_activities) == 1, ("There should be 1 new " - " activity in the user's activity stream, but found %i" % + " activity in the user's activity stream, but found %i" % len(user_new_activities)) assert user_new_activities[0] == activity @@ -2110,7 +2110,7 @@ def test_follow_user(self): followee_before['follower dashboard activity stream'], followee_after['follower dashboard activity stream'])) assert len(followee_new_activities) == 1, ("There should be 1 new " - " activity in the user's activity stream, but found %i" % + " activity in the user's activity stream, but found %i" % len(followee_new_activities)) assert followee_new_activities[0] == activity From 85310d44b79ddab590e687c23441b7e244243171 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Thu, 28 Jun 2012 18:05:33 +0100 Subject: [PATCH 111/155] [#2439] Update to docs --- doc/apiv3.rst | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/doc/apiv3.rst b/doc/apiv3.rst index a76c6be6e00..f9b5a580952 100644 --- a/doc/apiv3.rst +++ b/doc/apiv3.rst @@ -81,17 +81,13 @@ will result in the following parameters being sent to the This interface is *slightly* more limited than the POST interface because it doesn't allow passing nested dicts into the action be accessed. As a -consequence of this, currently the *resource_search*, *tag_search* and -*tag_autocomplete* actions are **limited** in their functionality. +consequence of this, currently the *resource_search* action is **limited** in +its functionality when accessed with a GET request. `resource_search`: This action is not currently usable via a GET request as it relies upon a nested dict of fields. -`tag_search` and `tag_autocomplete`: - The `fields` argument is not available when accessing this action with a - GET request. - Also, it is worth bearing this limitation in mind when creating your own actions via the `IActions` interface. From 47f2aa5c9fc355769a33d83eb9078fc4cfa5cc0e Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 29 Jun 2012 08:24:31 +0200 Subject: [PATCH 112/155] [#2345] Add new action API autodocs to CHANGELOG --- CHANGELOG.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 30175d6a057..4954bfb0852 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -17,6 +17,9 @@ v1.8 where you can see an activity stream from of all the users and datasets that you're following. There are also API calls for the follow features, see the Action API reference documentation. +* [#2345] New action API reference docs. The documentation for CKAN's Action + API has been rewritten, with each function and its arguments and return + values now individually documented. v1.7.1 2012-06-20 ================= From bfb48c0a1f0d3b6b518c41a9fe5d41bc0641aaf8 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Fri, 29 Jun 2012 10:46:34 +0100 Subject: [PATCH 113/155] 2601 Ensuring users not in a group cannot add datasets When publisher auth is enabled only users who are in a group/org should be able to add a dataset. --- ckan/logic/auth/publisher/create.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ckan/logic/auth/publisher/create.py b/ckan/logic/auth/publisher/create.py index ea371571b0a..0f395aa67b3 100644 --- a/ckan/logic/auth/publisher/create.py +++ b/ckan/logic/auth/publisher/create.py @@ -12,18 +12,20 @@ def package_create(context, data_dict=None): model = context['model'] user = context['user'] - userobj = model.User.get( user ) + userobj = model.User.get(user) - if userobj: + if userobj and len(userobj.get_groups()): return {'success': True} - return {'success': False, 'msg': 'You must be logged in to create a package'} + return {'success': False, + 'msg': _('You must be logged in and be within a group to create ' + 'a package')} def related_create(context, data_dict=None): model = context['model'] user = context['user'] - userobj = model.User.get( user ) + userobj = model.User.get(user) if userobj: return {'success': True} From 15a3ae9f04d765882a00c68aa7c934a6aa51d4f9 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Fri, 29 Jun 2012 10:53:31 +0100 Subject: [PATCH 114/155] rePEP8ification --- ckan/controllers/user.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index 2de826a3b94..30f901d796e 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -117,7 +117,8 @@ def me(self, locale=None): h.redirect_to(locale=locale, controller='user', action='login', id=None) user_ref = c.userobj.get_reference_preferred_for_uri() - h.redirect_to(locale=locale, controller='user', action='dashboard', id=user_ref) + h.redirect_to(locale=locale, controller='user', action='dashboard', + id=user_ref) def register(self, data=None, errors=None, error_summary=None): return self.new(data, errors, error_summary) @@ -447,7 +448,7 @@ def followers(self, id=None): def dashboard(self, id=None): context = {'model': model, 'session': model.Session, - 'user': c.user or c.author, 'for_view': True} - data_dict = {'id':id, 'user_obj':c.userobj} + 'user': c.user or c.author, 'for_view': True} + data_dict = {'id': id, 'user_obj': c.userobj} self._setup_template_variables(context, data_dict) return render('user/dashboard.html') From 54e8722bdb38ba66e9566c09c44c25edeb29aa07 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Fri, 29 Jun 2012 12:28:52 +0100 Subject: [PATCH 115/155] [#2439] Sanitise method parameters. And comment reason for making the list copy. Response to feedback from code-review. --- ckan/lib/search/query.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ckan/lib/search/query.py b/ckan/lib/search/query.py index feef9622ec1..39233423aeb 100644 --- a/ckan/lib/search/query.py +++ b/ckan/lib/search/query.py @@ -150,7 +150,10 @@ def run(self, query=None, terms=[], fields={}, facet_by=[], options=None, **kwar class TagSearchQuery(SearchQuery): """Search for tags.""" - def run(self, query=[], fields={}, options=None, **kwargs): + def run(self, query=None, fields=None, options=None, **kwargs): + query = [] if query is None else query + fields = {} if fields is None else fields + if options is None: options = QueryOptions(**kwargs) else: @@ -159,7 +162,7 @@ def run(self, query=[], fields={}, options=None, **kwargs): if isinstance(query, basestring): query = [query] - query = query[:] + query = query[:] # don't alter caller's query list. for field, value in fields.items(): if field in ('tag', 'tags'): query.append(value) From 5fa209a41e2b01faf629ce9fa91bb8c7eb172ed5 Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 29 Jun 2012 17:49:25 +0100 Subject: [PATCH 116/155] [#2389] Be more strict when checking for ES --- ckan/tests/functional/test_datastore.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ckan/tests/functional/test_datastore.py b/ckan/tests/functional/test_datastore.py index 2fb2d005460..1827b2adf70 100644 --- a/ckan/tests/functional/test_datastore.py +++ b/ckan/tests/functional/test_datastore.py @@ -111,7 +111,17 @@ def setup_class(cls): # Check if Elastic Search is available, otherwise skip tests try: - requests.get('http://%s/_status' % ELASTIC_SEARCH_HOST, timeout=2) + res = requests.get('http://%s/_status' % ELASTIC_SEARCH_HOST, timeout=2) + if res.status_code == 200: + try: + content = json.loads(res.content) + if not 'ok'in content or not '_shards' in content: + raise ValueError + except ValueError: + raise SkipTest('This does not look like Elastic Search, skipping...') + else: + raise SkipTest('Could not reach Elastic Search, skipping...') + except (requests.exceptions.RequestException, socket.timeout), e: raise SkipTest('Could not reach Elastic Search, skipping... (%r)' % e) From 71278f801b6bf734c7a9a5ddb745c556c0c3d866 Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 29 Jun 2012 17:51:57 +0100 Subject: [PATCH 117/155] [#2389] Yet more failing errors if 'error' was present on the JS strings --- ckan/tests/functional/test_authz.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ckan/tests/functional/test_authz.py b/ckan/tests/functional/test_authz.py index c70fa28982f..6b2e55dd62b 100644 --- a/ckan/tests/functional/test_authz.py +++ b/ckan/tests/functional/test_authz.py @@ -115,7 +115,9 @@ def _test_via_wui(self, action, user, entity_name, entity='dataset'): r = res.body r = r.replace('form_errors', '') - tests['error string'] = bool('error' not in r) + # Commenting as it seems a very ineffective way of checking for errors + # (e.g. tests fail if there is a JS string with the word 'error' on it) + # tests['error string'] = bool('error' not in r) tests['status'] = bool(res.status in (200, 201)) tests['0 packages found'] = bool(u'0 packages found' not in res) is_ok = False not in tests.values() From 528d27bf699b8df85edf8c55f7cb91b8fcf64c7c Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 2 Jul 2012 13:05:58 +0100 Subject: [PATCH 118/155] [#1678] Make synchronous search the default behaviour Unless already loaded or explicitly disabled via the `ckan.search.automatic_index` configuration option, the synchronous search plugin will be loaded during startup time. --- ckan/config/deployment.ini_tmpl | 7 ++++++- ckan/config/environment.py | 8 ++++++++ doc/configuration.rst | 17 +++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index d3c0199687f..d66d6543248 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -24,7 +24,7 @@ app_instance_uuid = ${app_instance_uuid} # List the names of CKAN extensions to activate. # Note: This line is required to be here for packaging, even if it is empty. -ckan.plugins = stats synchronous_search +ckan.plugins = stats # If you'd like to fine-tune the individual locations of the cache data dirs # for the Cache data, or the Session saves, un-comment the desired settings @@ -112,6 +112,11 @@ ckan.gravatar_default = identicon ## Solr support #solr_url = http://127.0.0.1:8983/solr +## Automatic indexing. Make all changes immediately available via the search +## after editing or creating a dataset. Default is true. If for some reason +## you need the indexing to occur asynchronously, set this option to 0. +# ckan.search.automatic_indexing = 1 + ## An 'id' for the site (using, for example, when creating entries in a common search index) ## If not specified derived from the site_url # ckan.site_id = ckan.net diff --git a/ckan/config/environment.py b/ckan/config/environment.py index bb46ea2db8a..8fca75b7548 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -19,6 +19,7 @@ import ckan.lib.search as search import ckan.lib.app_globals as app_globals +log = logging.getLogger(__name__) # Suppress benign warning 'Unbuilt egg for setuptools' warnings.simplefilter('ignore', UserWarning) @@ -122,6 +123,13 @@ def find_controller(self, controller): # load all CKAN plugins p.load_all(config) + # Load the synchronous search plugin, unless already loaded or + # explicitly disabled + if not 'synchronous_search' in config.get('ckan.plugins') and \ + asbool(config.get('ckan.search.automatic_indexing',True)): + log.debug('Loading the synchronous search plugin') + p.load('synchronous_search') + for plugin in p.PluginImplementations(p.IConfigurer): # must do update in place as this does not work: # config = plugin.update_config(config) diff --git a/doc/configuration.rst b/doc/configuration.rst index 9b753e480d9..ebb09845bdc 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -496,6 +496,23 @@ Optionally, ``solr_user`` and ``solr_password`` can also be configured to specif Note, if you change this value, you need to rebuild the search index. +.. index:: + single: ckan.search.automatic_indexing + +ckan.search.automatic_indexing +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.search.automatic_indexing = 1 + +Make all changes immediately available via the search after editing or +creating a dataset. Default is true. If for some reason you need the indexing +to occur asynchronously, set this option to 0. + +Note, this is equivalent to explicitly load the `synchronous_search` plugin. + + simple_search ^^^^^^^^^^^^^ From 67d7135ed8c2494279dbe00b67cfea20ed2bfc6c Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 2 Jul 2012 13:44:37 +0100 Subject: [PATCH 119/155] PEP8 environment.py --- ckan/config/environment.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/ckan/config/environment.py b/ckan/config/environment.py index 8fca75b7548..d388937ff33 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -24,6 +24,7 @@ # Suppress benign warning 'Unbuilt egg for setuptools' warnings.simplefilter('ignore', UserWarning) + class _Helpers(object): ''' Helper object giving access to template helpers stopping missing functions from causing template exceptions. Useful if @@ -94,13 +95,16 @@ def load_environment(global_conf, app_conf): from pylons.wsgiapp import PylonsApp import pkg_resources find_controller_generic = PylonsApp.find_controller + # This is from pylons 1.0 source, will monkey-patch into 0.9.7 def find_controller(self, controller): if controller in self.controller_classes: return self.controller_classes[controller] # Check to see if its a dotted name if '.' in controller or ':' in controller: - mycontroller = pkg_resources.EntryPoint.parse('x=%s' % controller).load(False) + mycontroller = pkg_resources \ + .EntryPoint \ + .parse('x=%s' % controller).load(False) self.controller_classes[controller] = mycontroller return mycontroller return find_controller_generic(self, controller) @@ -126,7 +130,7 @@ def find_controller(self, controller): # Load the synchronous search plugin, unless already loaded or # explicitly disabled if not 'synchronous_search' in config.get('ckan.plugins') and \ - asbool(config.get('ckan.search.automatic_indexing',True)): + asbool(config.get('ckan.search.automatic_indexing', True)): log.debug('Loading the synchronous search plugin') p.load('synchronous_search') @@ -160,11 +164,13 @@ def find_controller(self, controller): config['pylons.app_globals'] = app_globals.Globals() # add helper functions - restrict_helpers = asbool(config.get('ckan.restrict_template_vars', 'true')) + restrict_helpers = asbool( + config.get('ckan.restrict_template_vars', 'true')) helpers = _Helpers(h, restrict_helpers) config['pylons.h'] = helpers - ## redo template setup to use genshi.search_path (so remove std template setup) + # Redo template setup to use genshi.search_path + # (so remove std template setup) template_paths = [paths['templates'][0]] extra_template_paths = config.get('extra_template_paths', '') if extra_template_paths: @@ -173,6 +179,7 @@ def find_controller(self, controller): # Translator (i18n) translator = Translator(pylons.translator) + def template_loaded(template): translator.setup(template) @@ -203,8 +210,6 @@ def template_loaded(template): # # ################################################################# - - ''' This code is based on Genshi code @@ -273,11 +278,14 @@ def genshi_lookup_attr(cls, obj, key): # Setup the SQLAlchemy database engine # Suppress a couple of sqlalchemy warnings - warnings.filterwarnings('ignore', '^Unicode type received non-unicode bind param value', sqlalchemy.exc.SAWarning) - warnings.filterwarnings('ignore', "^Did not recognize type 'BIGINT' of column 'size'", sqlalchemy.exc.SAWarning) - warnings.filterwarnings('ignore', "^Did not recognize type 'tsvector' of column 'search_vector'", sqlalchemy.exc.SAWarning) + msgs = ['^Unicode type received non-unicode bind param value', + "^Did not recognize type 'BIGINT' of column 'size'", + "^Did not recognize type 'tsvector' of column 'search_vector'" + ] + for msg in msgs: + warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning) - ckan_db = os.environ.get('CKAN_DB') + ckan_db = os.environ.get('CKAN_DB') if ckan_db: config['sqlalchemy.url'] = ckan_db @@ -296,4 +304,3 @@ def genshi_lookup_attr(cls, obj, key): for plugin in p.PluginImplementations(p.IConfigurable): plugin.configure(config) - From 93931067fb9126c46cfbcec6a1a8f017aca1b31b Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 2 Jul 2012 15:06:07 +0200 Subject: [PATCH 120/155] Add credit to Sven in CHANGELOG --- CHANGELOG.txt | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 4954bfb0852..3a117515ddf 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -9,14 +9,17 @@ v1.8 Some of CKAN's dependencies have been updated and some removed. * Requirements have been updated see doc/install-from-source.rst users will need to do a new pip install (#2592) -* [#2304,#2305] New 'follow' feature. You'll now see a 'Followers' tab on user - and dataset pages, where you can see how many users are following that user - or dataset. If you're logged in, you'll see a 'Follow' button on the pages - of datasets and other users that you can click to follow them. Also when - logged in, if you go to your own user page you'll see a new 'Dashboard' tab - where you can see an activity stream from of all the users and datasets that - you're following. There are also API calls for the follow features, see the - Action API reference documentation. +* [#2304] New 'follow' feature. You'll now see a 'Followers' tab on user and + dataset pages, where you can see how many users are following that user or + dataset. If you're logged in, you'll see a 'Follow' button on the pages of + datasets and other users that you can click to follow them. There are also + API calls for the follow features, see the Action API reference + documentation. +* [#2305] New user dashboards, implemented by Sven R. Kunze + (https://github.com/kunsv) as part of his Masters thesis. When logged in, if + you go to your own user page you'll see a new 'Dashboard' tab where you can + see an activity stream from of all the users and datasets that you're + following. * [#2345] New action API reference docs. The documentation for CKAN's Action API has been rewritten, with each function and its arguments and return values now individually documented. From af1f41fb48fc1e8634455b2fa3b464d28de6213d Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 2 Jul 2012 16:00:26 +0200 Subject: [PATCH 121/155] [#2290] Mark facet titles in search pages for translation --- ckan/templates/package/search.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/templates/package/search.html b/ckan/templates/package/search.html index 1414640dd1c..01a403bc348 100644 --- a/ckan/templates/package/search.html +++ b/ckan/templates/package/search.html @@ -21,9 +21,9 @@

    Add a dataset

    • - ${facet_div('tags', 'Tags')} - ${facet_div('res_format', 'Resource Formats')} - ${facet_div('groups', 'Groups')} + ${facet_div('tags', _('Tags'))} + ${facet_div('res_format', _('Resource Formats'))} + ${facet_div('groups', _('Groups'))}

  • Other access

    From 33faf3e512fa888414e9b191dadb3835fb5e232f Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 2 Jul 2012 16:02:18 +0200 Subject: [PATCH 122/155] Mark facet titles on group read page for translation --- ckan/templates/group/read.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/templates/group/read.html b/ckan/templates/group/read.html index c4744020b54..c6c5839cc61 100644 --- a/ckan/templates/group/read.html +++ b/ckan/templates/group/read.html @@ -25,8 +25,8 @@

    Administrators

- ${facet_div('tags', 'Tags')} - ${facet_div('res_format', 'Resource Formats')} + ${facet_div('tags', _('Tags'))} + ${facet_div('res_format', _('Resource Formats'))} From 4594319109fd4e2cd8f152216ae9214e22a8c2ca Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 2 Jul 2012 15:38:18 +0200 Subject: [PATCH 123/155] Add strings from core extensions to pot file --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a8f8297e035..74187be3b0c 100644 --- a/setup.py +++ b/setup.py @@ -45,8 +45,13 @@ }), ('public/**', 'ignore', None), ], - 'ckanext/stats/templates': [ + 'ckanext': [ + ('**.py', 'python', None), ('**.html', 'genshi', None), + ('multilingual/solr/*.txt', 'ignore', None), + ('**.txt', 'genshi', { + 'template_class': 'genshi.template:TextTemplate' + }), ]}, entry_points=""" [nose.plugins.0.10] From fb782e2043d58c7015eb17ee6fe236316dc79ea9 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 2 Jul 2012 15:38:52 +0200 Subject: [PATCH 124/155] Fix a string that had a lot of whitespace in it --- ckanext/organizations/controllers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/organizations/controllers.py b/ckanext/organizations/controllers.py index fbe6e25e464..371b2141d98 100644 --- a/ckanext/organizations/controllers.py +++ b/ckanext/organizations/controllers.py @@ -95,8 +95,8 @@ def apply(self, id=None, data=None, errors=None, error_summary=None): def _add_users( self, group, parameters ): if not group: - h.flash_error(_("There was a problem with your submission, \ - please correct it and try again")) + h.flash_error(_("There was a problem with your submission, " + "please correct it and try again")) errors = {"reason": ["No reason was supplied"]} return self.apply(group.id, errors=errors, error_summary=action.error_summary(errors)) From 58d054970a1ff108678ae8ee3124c7f27eca224a Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Mon, 25 Jun 2012 17:42:38 +0100 Subject: [PATCH 125/155] Dictize results list of resource_search action. Looks like this action wasn't even working! --- ckan/logic/action/get.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index f15525bd768..cb215d0546b 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1240,7 +1240,8 @@ def resource_search(context, data_dict): else: results.append(result) - return {'count': count, 'results': results} + return {'count': count, + 'results': model_dictize.resource_list_dictize(results, context)} def _tag_search(context, data_dict): model = context['model'] From f386122015e1b8408b1f94046d8422eb2d17ac7a Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Mon, 25 Jun 2012 17:44:20 +0100 Subject: [PATCH 126/155] Docstring for resource_search action. --- ckan/logic/action/get.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index cb215d0546b..1737b76bb17 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1178,18 +1178,24 @@ def package_search(context, data_dict): def resource_search(context, data_dict): ''' + Searches for resources satisfying a given search criteria. + + It returns a dictionary with 2 fields: ``count`` and ``results``. The + ``count`` field contains the total number of Resources found without the + limit or query parameters having an effect. The ``results`` field is a + list of dictized Resource objects. :param fields: :type fields: - :param order_by: - :type order_by: - :param offset: - :type offset: - :param limit: - :type limit: - - :returns: - :rtype: + :param order_by: A field on the Resource model that orders the results. + :type order_by: string + :param offset: Apply an offset to the query. + :type offset: int + :param limit: Apply a limit to the query. + :type limit: int + + :returns: A dictionary with a ``count`` field, and a ``results`` field. + :rtype: dict ''' model = context['model'] From ebd9bc613ea4cea5617998cf2c669ff3f267f4dd Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 15:43:55 +0100 Subject: [PATCH 127/155] Add 'q' parameter to resource_search action. This deprecates the existing 'fields' parameter, which is not compatible with GET-ing the resource_search action. --- ckan/logic/action/get.py | 56 +++++++++++++++++++++++++++-- ckan/tests/logic/test_action.py | 63 +++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 3 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 1737b76bb17..37d0ac0a37e 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1185,8 +1185,52 @@ def resource_search(context, data_dict): limit or query parameters having an effect. The ``results`` field is a list of dictized Resource objects. - :param fields: - :type fields: + The 'q' parameter is a required field. It is a string of the form + ``{field}:{term}`` or a list of strings, each of the same form. Within + each string, ``{field}`` is a field or extra field on the Resource domain + object. + + If ``{field}`` is ``"hash"``, then an attempt is made to match the + `{term}` as a *prefix* of the ``Resource.hash`` field. + + If ``{field}`` is an extra field, then an attempt is made to match against + the extra fields stored against the Resource. + + Note: The search is limited to search against extra fields declared in + the config setting ``ckan.extra_resource_fields``. + + Note: Due to a Resource's extra fields being stored as a json blob, the + match is made against the json string representation. As such, false + positives may occur: + + If the search criteria is: :: + + query = "field1:term1" + + Then a json blob with the string representation of: :: + + {"field1": "foo", "field2": "term1"} + + will match the search criteria! This is a known short-coming of this + approach. + + All matches are made ignoring case; and apart from the ``"hash"`` field, + a term matches if it is a substring of the field's value. + + Finally, when specifying more than one search criteria, the criteria are + AND-ed together. + + The ``order`` parameter is used to control the ordering of the results. + Currently only ordering one field is available, and in ascending order + only. + + The ``fields`` parameter is deprecated as it is not compatible with calling + this action with a GET request to the action API. + + :param query: The search criteria. See above for description. + :type query: string or list of strings of the form "{field}:{term1}" + :param fields: Deprecated + :type fields: dict of fields to search terms. :param order_by: A field on the Resource model that orders the results. :type order_by: string :param offset: Apply an offset to the query. @@ -1201,15 +1245,21 @@ def resource_search(context, data_dict): model = context['model'] session = context['session'] + query = _get_or_bust(data_dict, 'query') fields = _get_or_bust(data_dict, 'fields') order_by = data_dict.get('order_by') offset = data_dict.get('offset') limit = data_dict.get('limit') + if isinstance(query, basestring): + query = [query] + + # TODO: escape ':' with '\:' + fields = dict(pair.split(":", 1) for pair in query) + # TODO: should we check for user authentication first? q = model.Session.query(model.Resource) resource_fields = model.Resource.get_columns() - for field, terms in fields.items(): if isinstance(terms, basestring): terms = terms.split() diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index d104c5d3e94..c8dc581c341 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -1023,6 +1023,69 @@ def test_41_missing_action(self): except KeyError: assert True + def test_42_resource_search_with_single_field_query(self): + request_body = { + 'q': ["description:index"], + 'fields': [] + } + postparams = json.dumps(request_body) + response = self.app.post('/api/action/resource_search', + params=postparams) + result = json.loads(response.body)['result']['results'] + + ## Due to the side-effect of previously run tests, there may be extra + ## resources in the results. So just check that each found Resource + ## matches the search criteria + assert result is not [] + for resource in result: + assert "index" in resource['description'].lower() + + def test_42_resource_search_across_multiple_fields(self): + request_body = { + 'q': ["description:index", "format:json"], + 'fields': [] + } + postparams = json.dumps(request_body) + response = self.app.post('/api/action/resource_search', + params=postparams) + result = json.loads(response.body)['result']['results'] + + ## Due to the side-effect of previously run tests, there may be extra + ## resources in the results. So just check that each found Resource + ## matches the search criteria + assert result is not [] + for resource in result: + assert "index" in resource['description'].lower() + assert "json" in resource['format'].lower() + + def test_42_resource_search_test_percentage_is_escaped(self): + pass + + def test_42_resource_search_escaped_colons(self): + pass + + def test_42_resource_search_fields_parameter_still_accepted(self): + '''The fields parameter is deprecated, but check it still works. + + Remove this test when removing the fields parameter. (#????) + ''' + request_body = { + 'fields': [("description", "index")], + 'q': [], + } + + postparams = json.dumps(request_body) + response = self.app.post('/api/action/resource_search', + params=postparams) + result = json.loads(response.body)['result']['results'] + + ## Due to the side-effect of previously run tests, there may be extra + ## resources in the results. So just check that each found Resource + ## matches the search criteria + assert result is not [] + for resource in result: + assert "index" in resource['description'].lower() + class TestActionTermTranslation(WsgiAppCase): @classmethod From cc070212a450b7c716cf79e81b29ba54301709be Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 16:52:49 +0100 Subject: [PATCH 128/155] Ensure legacy parameter, fields, still works. Keep the old fields `parameter's` behaviour, but encourage use of the new `query` parameter. --- ckan/logic/action/get.py | 38 +++++++++++++++++++++++++-------- ckan/tests/logic/test_action.py | 9 +++----- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 37d0ac0a37e..c0657748ce9 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1245,24 +1245,44 @@ def resource_search(context, data_dict): model = context['model'] session = context['session'] - query = _get_or_bust(data_dict, 'query') - fields = _get_or_bust(data_dict, 'fields') + # Allow either query or fields parameter to be given, but not both. + # Once ``fields`` parameter is dropped, this can be made simpler. + # The result of all this gumpf is to populate the local `fields` variable + # with mappings from field names to list of search terms, or a single + # search-term string. + query = data_dict.get('query') + fields = data_dict.get('fields') + if query is None and fields is None: + raise ValidationError({'query': _('Missing value')}) + elif query is not None and fields is not None: + raise ValidationError( + {'fields': _('Do not specify if using "query" parameter')}) + elif query is not None: + if isinstance(query, basestring): + query = [query] + + # TODO: escape ':' with '\:' + fields = dict(pair.split(":", 1) for pair in query) + else: + # legacy fields paramter would split string terms + # maintain that behaviour + split_terms = {} + for field, terms in fields.items(): + if isinstance(terms, basestring): + terms = terms.split() + split_terms[field] = terms + fields = split_terms + order_by = data_dict.get('order_by') offset = data_dict.get('offset') limit = data_dict.get('limit') - if isinstance(query, basestring): - query = [query] - - # TODO: escape ':' with '\:' - fields = dict(pair.split(":", 1) for pair in query) - # TODO: should we check for user authentication first? q = model.Session.query(model.Resource) resource_fields = model.Resource.get_columns() for field, terms in fields.items(): if isinstance(terms, basestring): - terms = terms.split() + terms = [terms] if field not in resource_fields: raise search.SearchError('Field "%s" not recognised in Resource search.' % field) for term in terms: diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index c8dc581c341..acb4104d964 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -1025,8 +1025,7 @@ def test_41_missing_action(self): def test_42_resource_search_with_single_field_query(self): request_body = { - 'q': ["description:index"], - 'fields': [] + 'query': ["description:index"], } postparams = json.dumps(request_body) response = self.app.post('/api/action/resource_search', @@ -1042,8 +1041,7 @@ def test_42_resource_search_with_single_field_query(self): def test_42_resource_search_across_multiple_fields(self): request_body = { - 'q': ["description:index", "format:json"], - 'fields': [] + 'query': ["description:index", "format:json"], } postparams = json.dumps(request_body) response = self.app.post('/api/action/resource_search', @@ -1070,8 +1068,7 @@ def test_42_resource_search_fields_parameter_still_accepted(self): Remove this test when removing the fields parameter. (#????) ''' request_body = { - 'fields': [("description", "index")], - 'q': [], + 'fields': {"description": "index"}, } postparams = json.dumps(request_body) From f8450f4f2b11308e340c83bd78d114e513717615 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 17:42:28 +0100 Subject: [PATCH 129/155] Special chars are escaped in search terms. --- ckan/logic/action/get.py | 1 + ckan/tests/logic/test_action.py | 23 +++++++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index c0657748ce9..5c03b07b63e 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1286,6 +1286,7 @@ def resource_search(context, data_dict): if field not in resource_fields: raise search.SearchError('Field "%s" not recognised in Resource search.' % field) for term in terms: + term = misc.escape_sql_like_special_characters(term) model_attr = getattr(model.Resource, field) if field == 'hash': q = q.filter(model_attr.ilike(unicode(term) + '%')) diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index acb4104d964..330238aa661 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -1031,11 +1031,12 @@ def test_42_resource_search_with_single_field_query(self): response = self.app.post('/api/action/resource_search', params=postparams) result = json.loads(response.body)['result']['results'] + count = json.loads(response.body)['result']['count'] ## Due to the side-effect of previously run tests, there may be extra ## resources in the results. So just check that each found Resource ## matches the search criteria - assert result is not [] + assert count > 0 for resource in result: assert "index" in resource['description'].lower() @@ -1047,20 +1048,29 @@ def test_42_resource_search_across_multiple_fields(self): response = self.app.post('/api/action/resource_search', params=postparams) result = json.loads(response.body)['result']['results'] + count = json.loads(response.body)['result']['count'] ## Due to the side-effect of previously run tests, there may be extra ## resources in the results. So just check that each found Resource ## matches the search criteria - assert result is not [] + assert count > 0 for resource in result: assert "index" in resource['description'].lower() assert "json" in resource['format'].lower() def test_42_resource_search_test_percentage_is_escaped(self): - pass + request_body = { + 'query': ["description:index%"], + } + postparams = json.dumps(request_body) + response = self.app.post('/api/action/resource_search', + params=postparams) + count = json.loads(response.body)['result']['count'] - def test_42_resource_search_escaped_colons(self): - pass + # There shouldn't be any results. If the '%' character wasn't + # escaped correctly, then the search would match because of the + # unescaped wildcard. + assert count is 0 def test_42_resource_search_fields_parameter_still_accepted(self): '''The fields parameter is deprecated, but check it still works. @@ -1075,11 +1085,12 @@ def test_42_resource_search_fields_parameter_still_accepted(self): response = self.app.post('/api/action/resource_search', params=postparams) result = json.loads(response.body)['result']['results'] + count = json.loads(response.body)['result']['count'] ## Due to the side-effect of previously run tests, there may be extra ## resources in the results. So just check that each found Resource ## matches the search criteria - assert result is not [] + assert count > 0 for resource in result: assert "index" in resource['description'].lower() From aeef82cbf16656308730b3e3b9121a4828b526c2 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 17:53:41 +0100 Subject: [PATCH 130/155] Useful validation errors for resource_search --- ckan/logic/action/get.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 5c03b07b63e..69ad1cd4949 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1260,9 +1260,11 @@ def resource_search(context, data_dict): elif query is not None: if isinstance(query, basestring): query = [query] - - # TODO: escape ':' with '\:' - fields = dict(pair.split(":", 1) for pair in query) + try: + fields = dict(pair.split(":", 1) for pair in query) + except ValueError: + raise ValidationError( + {'query': _('Must be {field}:{value} pair(s)')}) else: # legacy fields paramter would split string terms # maintain that behaviour @@ -1284,7 +1286,10 @@ def resource_search(context, data_dict): if isinstance(terms, basestring): terms = [terms] if field not in resource_fields: - raise search.SearchError('Field "%s" not recognised in Resource search.' % field) + raise ValidationError( + {'query': + _('Field "{field}" not recognised in resource_search.')\ + .format(field=field)}) for term in terms: term = misc.escape_sql_like_special_characters(term) model_attr = getattr(model.Resource, field) From 966ab39183351701656786d63a193bf79898c748 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 17:54:23 +0100 Subject: [PATCH 131/155] Remove unused session object. --- ckan/logic/action/get.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 69ad1cd4949..fc7ca2a1090 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1243,7 +1243,6 @@ def resource_search(context, data_dict): ''' model = context['model'] - session = context['session'] # Allow either query or fields parameter to be given, but not both. # Once ``fields`` parameter is dropped, this can be made simpler. From 79505e541263eadce6f7245e6548240499413ca1 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 18:00:48 +0100 Subject: [PATCH 132/155] General tidy of resource_search action. --- ckan/logic/action/get.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index fc7ca2a1090..82963437153 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1244,18 +1244,21 @@ def resource_search(context, data_dict): ''' model = context['model'] - # Allow either query or fields parameter to be given, but not both. - # Once ``fields`` parameter is dropped, this can be made simpler. + # Allow either the `query` or `fields` parameter to be given, but not both. + # Once `fields` parameter is dropped, this can be made simpler. # The result of all this gumpf is to populate the local `fields` variable # with mappings from field names to list of search terms, or a single # search-term string. query = data_dict.get('query') fields = data_dict.get('fields') + if query is None and fields is None: raise ValidationError({'query': _('Missing value')}) + elif query is not None and fields is not None: raise ValidationError( {'fields': _('Do not specify if using "query" parameter')}) + elif query is not None: if isinstance(query, basestring): query = [query] @@ -1264,9 +1267,10 @@ def resource_search(context, data_dict): except ValueError: raise ValidationError( {'query': _('Must be {field}:{value} pair(s)')}) + else: - # legacy fields paramter would split string terms - # maintain that behaviour + # The legacy fields paramter splits string terms. + # So maintain that behaviour split_terms = {} for field, terms in fields.items(): if isinstance(terms, basestring): @@ -1282,18 +1286,29 @@ def resource_search(context, data_dict): q = model.Session.query(model.Resource) resource_fields = model.Resource.get_columns() for field, terms in fields.items(): + if isinstance(terms, basestring): terms = [terms] + if field not in resource_fields: raise ValidationError( {'query': _('Field "{field}" not recognised in resource_search.')\ .format(field=field)}) + for term in terms: + + # prevent pattern injection term = misc.escape_sql_like_special_characters(term) + model_attr = getattr(model.Resource, field) + + # Treat the has field separately, see docstring. if field == 'hash': q = q.filter(model_attr.ilike(unicode(term) + '%')) + + # Resource extras are stored in a json blob. So searching for + # matching fields is a bit trickier. See the docstring. elif field in model.Resource.get_extra_columns(): model_attr = getattr(model.Resource, 'extras') @@ -1302,6 +1317,8 @@ def resource_search(context, data_dict): model_attr.ilike(u'''%%"%s": "%%%s%%"}''' % (field, term)) ) q = q.filter(like) + + # Just a regular field else: q = q.filter(model_attr.ilike('%' + unicode(term) + '%')) From d4ea63a2b9c2f5dd339879269fb366f847b3ab76 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 18:05:41 +0100 Subject: [PATCH 133/155] Add reference to ticket number --- ckan/tests/logic/test_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index 330238aa661..c9b672126b6 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -1075,7 +1075,7 @@ def test_42_resource_search_test_percentage_is_escaped(self): def test_42_resource_search_fields_parameter_still_accepted(self): '''The fields parameter is deprecated, but check it still works. - Remove this test when removing the fields parameter. (#????) + Remove this test when removing the fields parameter. (#2603) ''' request_body = { 'fields': {"description": "index"}, From dcca75713e44763645c2190cd531b30f8c99ff31 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 18:18:15 +0100 Subject: [PATCH 134/155] Add log message warning of deprecated parameter. Obviously this won't indicate to the action api users, only when in library access. --- ckan/logic/action/get.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 82963437153..b8825331da9 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1269,6 +1269,9 @@ def resource_search(context, data_dict): {'query': _('Must be {field}:{value} pair(s)')}) else: + log.warning('Use of the "fields" parameter in resource_search is ' + 'deprecated. Use the "query" parameter instead') + # The legacy fields paramter splits string terms. # So maintain that behaviour split_terms = {} From 575a96470a9278dc660144743999b9c2849e114d Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 18:50:28 +0100 Subject: [PATCH 135/155] Context determines resource_search's result type. This is provided as it allows the search api to continue to use this action, and still provide the required result-type for the action api. --- ckan/lib/search/query.py | 6 +++++- ckan/logic/action/get.py | 25 ++++++++++++++++++++----- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/ckan/lib/search/query.py b/ckan/lib/search/query.py index a3b7adf35fe..4d11961d01d 100644 --- a/ckan/lib/search/query.py +++ b/ckan/lib/search/query.py @@ -186,7 +186,11 @@ def run(self, fields={}, options=None, **kwargs): else: options.update(kwargs) - context = {'model':model, 'session': model.Session} + context = { + 'model':model, + 'session': model.Session, + 'search_query': True, + } data_dict = { 'fields': fields, 'offset': options.get('offset'), diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index b8825331da9..d84cc1cd759 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1227,6 +1227,11 @@ def resource_search(context, data_dict): The ``fields`` parameter is deprecated as it is not compatible with calling this action with a GET request to the action API. + The context may contain a flag, `search_query`, which if True will make + this action behave as if being used by the internal search api. ie - the + results will not be dictized, and SearchErrors are thrown for bad search + queries (rather than ValidationErrors). + :param query: The search criteria. See above for description. :type query: string or list of strings of the form "{field}:{term1}" :param fields: Deprecated @@ -1294,10 +1299,16 @@ def resource_search(context, data_dict): terms = [terms] if field not in resource_fields: - raise ValidationError( - {'query': - _('Field "{field}" not recognised in resource_search.')\ - .format(field=field)}) + msg = _('Field "{field}" not recognised in resource_search.')\ + .format(field=field) + + # Running in the context of the internal search api. + if context.get('search_query', False): + raise search.SearchError(msg) + + # Otherwise, assume we're in the context of an external api + # and need to provide meaningful external error messages. + raise ValidationError({'query': msg}) for term in terms: @@ -1341,8 +1352,12 @@ def resource_search(context, data_dict): else: results.append(result) + # If run in the context of a search query, then don't dictize the results. + if not context.get('search_query', False): + results = model_dictize.resource_list_dictize(results, context) + return {'count': count, - 'results': model_dictize.resource_list_dictize(results, context)} + 'results': results} def _tag_search(context, data_dict): model = context['model'] From aad834c25ce6f28cd719a59d3d89f5cc79ab533c Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 19:13:22 +0100 Subject: [PATCH 136/155] Test resource_search works with a GET request --- ckan/tests/logic/test_action.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index c9b672126b6..3b3566fecf7 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -1094,6 +1094,21 @@ def test_42_resource_search_fields_parameter_still_accepted(self): for resource in result: assert "index" in resource['description'].lower() + def test_42_resource_search_accessible_via_get_request(self): + response = self.app.get('/api/action/resource_search' + '?query=description:index&query=format:json') + + result = json.loads(response.body)['result']['results'] + count = json.loads(response.body)['result']['count'] + + ## Due to the side-effect of previously run tests, there may be extra + ## resources in the results. So just check that each found Resource + ## matches the search criteria + assert count > 0 + for resource in result: + assert "index" in resource['description'].lower() + assert "json" in resource['format'].lower() + class TestActionTermTranslation(WsgiAppCase): @classmethod From f4b6a025618d4ae874f4ffb452365cc6301b4aa6 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Wed, 27 Jun 2012 10:54:35 +0100 Subject: [PATCH 137/155] Use the new 'query' parameter in resource_search instead of the now deprecated 'fields' paramter. --- ckan/lib/search/query.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ckan/lib/search/query.py b/ckan/lib/search/query.py index 4d11961d01d..1380e531bb1 100644 --- a/ckan/lib/search/query.py +++ b/ckan/lib/search/query.py @@ -191,8 +191,18 @@ def run(self, fields={}, options=None, **kwargs): 'session': model.Session, 'search_query': True, } + + # Transform fields into structure required by the resource_search + # action. + query = [] + for field, terms in fields.items(): + if isinstance(terms, basestring): + terms = terms.split() + for term in terms: + query.append(':'.join([field, term])) + data_dict = { - 'fields': fields, + 'query': query, 'offset': options.get('offset'), 'limit': options.get('limit'), 'order_by': options.get('order_by') From 80781b39423084f33cacd981e8e88e9a69c4d1ce Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Wed, 27 Jun 2012 10:56:36 +0100 Subject: [PATCH 138/155] Clarify translatable string. ``{field}`` is usually used to represent a substitution. --- ckan/logic/action/get.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index d84cc1cd759..829a7906828 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1271,7 +1271,7 @@ def resource_search(context, data_dict): fields = dict(pair.split(":", 1) for pair in query) except ValueError: raise ValidationError( - {'query': _('Must be {field}:{value} pair(s)')}) + {'query': _('Must be : pair(s)')}) else: log.warning('Use of the "fields" parameter in resource_search is ' From 3e00ffea420bd11af64b0d2d417e3c6100b71390 Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 2 Jul 2012 13:05:58 +0100 Subject: [PATCH 139/155] Make synchronous search the default behaviour Unless already loaded or explicitly disabled via the `ckan.search.automatic_index` configuration option, the synchronous search plugin will be loaded during startup time. --- ckan/config/deployment.ini_tmpl | 7 ++++++- ckan/config/environment.py | 8 ++++++++ doc/configuration.rst | 17 +++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index d3c0199687f..d66d6543248 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -24,7 +24,7 @@ app_instance_uuid = ${app_instance_uuid} # List the names of CKAN extensions to activate. # Note: This line is required to be here for packaging, even if it is empty. -ckan.plugins = stats synchronous_search +ckan.plugins = stats # If you'd like to fine-tune the individual locations of the cache data dirs # for the Cache data, or the Session saves, un-comment the desired settings @@ -112,6 +112,11 @@ ckan.gravatar_default = identicon ## Solr support #solr_url = http://127.0.0.1:8983/solr +## Automatic indexing. Make all changes immediately available via the search +## after editing or creating a dataset. Default is true. If for some reason +## you need the indexing to occur asynchronously, set this option to 0. +# ckan.search.automatic_indexing = 1 + ## An 'id' for the site (using, for example, when creating entries in a common search index) ## If not specified derived from the site_url # ckan.site_id = ckan.net diff --git a/ckan/config/environment.py b/ckan/config/environment.py index bb46ea2db8a..8fca75b7548 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -19,6 +19,7 @@ import ckan.lib.search as search import ckan.lib.app_globals as app_globals +log = logging.getLogger(__name__) # Suppress benign warning 'Unbuilt egg for setuptools' warnings.simplefilter('ignore', UserWarning) @@ -122,6 +123,13 @@ def find_controller(self, controller): # load all CKAN plugins p.load_all(config) + # Load the synchronous search plugin, unless already loaded or + # explicitly disabled + if not 'synchronous_search' in config.get('ckan.plugins') and \ + asbool(config.get('ckan.search.automatic_indexing',True)): + log.debug('Loading the synchronous search plugin') + p.load('synchronous_search') + for plugin in p.PluginImplementations(p.IConfigurer): # must do update in place as this does not work: # config = plugin.update_config(config) diff --git a/doc/configuration.rst b/doc/configuration.rst index 9b753e480d9..ebb09845bdc 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -496,6 +496,23 @@ Optionally, ``solr_user`` and ``solr_password`` can also be configured to specif Note, if you change this value, you need to rebuild the search index. +.. index:: + single: ckan.search.automatic_indexing + +ckan.search.automatic_indexing +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.search.automatic_indexing = 1 + +Make all changes immediately available via the search after editing or +creating a dataset. Default is true. If for some reason you need the indexing +to occur asynchronously, set this option to 0. + +Note, this is equivalent to explicitly load the `synchronous_search` plugin. + + simple_search ^^^^^^^^^^^^^ From 08cba3144db70784f90c381a2139c8c035f2b63e Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 2 Jul 2012 13:44:37 +0100 Subject: [PATCH 140/155] PEP8 environment.py --- ckan/config/environment.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/ckan/config/environment.py b/ckan/config/environment.py index 8fca75b7548..d388937ff33 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -24,6 +24,7 @@ # Suppress benign warning 'Unbuilt egg for setuptools' warnings.simplefilter('ignore', UserWarning) + class _Helpers(object): ''' Helper object giving access to template helpers stopping missing functions from causing template exceptions. Useful if @@ -94,13 +95,16 @@ def load_environment(global_conf, app_conf): from pylons.wsgiapp import PylonsApp import pkg_resources find_controller_generic = PylonsApp.find_controller + # This is from pylons 1.0 source, will monkey-patch into 0.9.7 def find_controller(self, controller): if controller in self.controller_classes: return self.controller_classes[controller] # Check to see if its a dotted name if '.' in controller or ':' in controller: - mycontroller = pkg_resources.EntryPoint.parse('x=%s' % controller).load(False) + mycontroller = pkg_resources \ + .EntryPoint \ + .parse('x=%s' % controller).load(False) self.controller_classes[controller] = mycontroller return mycontroller return find_controller_generic(self, controller) @@ -126,7 +130,7 @@ def find_controller(self, controller): # Load the synchronous search plugin, unless already loaded or # explicitly disabled if not 'synchronous_search' in config.get('ckan.plugins') and \ - asbool(config.get('ckan.search.automatic_indexing',True)): + asbool(config.get('ckan.search.automatic_indexing', True)): log.debug('Loading the synchronous search plugin') p.load('synchronous_search') @@ -160,11 +164,13 @@ def find_controller(self, controller): config['pylons.app_globals'] = app_globals.Globals() # add helper functions - restrict_helpers = asbool(config.get('ckan.restrict_template_vars', 'true')) + restrict_helpers = asbool( + config.get('ckan.restrict_template_vars', 'true')) helpers = _Helpers(h, restrict_helpers) config['pylons.h'] = helpers - ## redo template setup to use genshi.search_path (so remove std template setup) + # Redo template setup to use genshi.search_path + # (so remove std template setup) template_paths = [paths['templates'][0]] extra_template_paths = config.get('extra_template_paths', '') if extra_template_paths: @@ -173,6 +179,7 @@ def find_controller(self, controller): # Translator (i18n) translator = Translator(pylons.translator) + def template_loaded(template): translator.setup(template) @@ -203,8 +210,6 @@ def template_loaded(template): # # ################################################################# - - ''' This code is based on Genshi code @@ -273,11 +278,14 @@ def genshi_lookup_attr(cls, obj, key): # Setup the SQLAlchemy database engine # Suppress a couple of sqlalchemy warnings - warnings.filterwarnings('ignore', '^Unicode type received non-unicode bind param value', sqlalchemy.exc.SAWarning) - warnings.filterwarnings('ignore', "^Did not recognize type 'BIGINT' of column 'size'", sqlalchemy.exc.SAWarning) - warnings.filterwarnings('ignore', "^Did not recognize type 'tsvector' of column 'search_vector'", sqlalchemy.exc.SAWarning) + msgs = ['^Unicode type received non-unicode bind param value', + "^Did not recognize type 'BIGINT' of column 'size'", + "^Did not recognize type 'tsvector' of column 'search_vector'" + ] + for msg in msgs: + warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning) - ckan_db = os.environ.get('CKAN_DB') + ckan_db = os.environ.get('CKAN_DB') if ckan_db: config['sqlalchemy.url'] = ckan_db @@ -296,4 +304,3 @@ def genshi_lookup_attr(cls, obj, key): for plugin in p.PluginImplementations(p.IConfigurable): plugin.configure(config) - From 63d82ceaea33277b21be3d336f1cd29d98959704 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 28 Jun 2012 18:53:24 +0200 Subject: [PATCH 141/155] Update several dependencies Replaced PyUtilib==4.0.2848 with pyutilib.component.core==4.5.3, this avoids pulling in all the pyutilib packages and the packages they depend on. Updated webhelpers, solrpy, formalchemy, markupsafe, tempita, routes and paste to the latest versions from pypi --- pip-requirements.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pip-requirements.txt b/pip-requirements.txt index 8a9fc42ce87..98f4a9d7150 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -5,23 +5,23 @@ Genshi==0.6 sqlalchemy-migrate==0.7.1 sqlalchemy==0.7.3 -webhelpers==1.2 -PyUtilib==4.0.2848 +webhelpers==1.3 +pyutilib.component.core==4.5.3 -e git+https://github.com/okfn/vdm.git@vdm-0.11#egg=vdm -solrpy==0.9.4 -formalchemy==1.4.1 +solrpy==0.9.5 +formalchemy==1.4.2 pairtree==0.7.1-T ofs==0.4.1 apachemiddleware==0.1.1 -markupsafe==0.9.2 +markupsafe==0.15 babel==0.9.4 psycopg2==2.0.13 webob==1.0.8 Pylons==0.9.7 repoze.who==1.0.19 -tempita==0.4 +tempita==0.5.1 zope.interface==3.5.3 repoze.who.plugins.openid==0.5.3 repoze.who-friendlyform==1.0.8 -routes==1.12 -paste==1.7.2 +routes==1.13 +paste==1.7.5.1 From a8dab115f0e94b7f475d15ffd6897bd03a6f81b4 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 28 Jun 2012 19:23:03 +0200 Subject: [PATCH 142/155] Update babel, psyco and zope dependencies --- pip-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pip-requirements.txt b/pip-requirements.txt index 98f4a9d7150..3156743d773 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -14,13 +14,13 @@ pairtree==0.7.1-T ofs==0.4.1 apachemiddleware==0.1.1 markupsafe==0.15 -babel==0.9.4 -psycopg2==2.0.13 +babel==0.9.6 +psycopg2==2.4.5 webob==1.0.8 Pylons==0.9.7 repoze.who==1.0.19 tempita==0.5.1 -zope.interface==3.5.3 +zope.interface==4.0.1 repoze.who.plugins.openid==0.5.3 repoze.who-friendlyform==1.0.8 routes==1.13 From 755431512cf806b3af77a3942b613da369f719f8 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 29 Jun 2012 06:18:03 +0000 Subject: [PATCH 143/155] Upgrade sqlalchemy-migrate to 0.7.2 (latest) --- pip-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pip-requirements.txt b/pip-requirements.txt index 3156743d773..5db1fc4c09a 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -3,7 +3,7 @@ # like: pip install -r pip-requirements.txt. See the Install from Source # instructions in CKAN's documentation for full installation instructions. Genshi==0.6 -sqlalchemy-migrate==0.7.1 +sqlalchemy-migrate==0.7.2 sqlalchemy==0.7.3 webhelpers==1.3 pyutilib.component.core==4.5.3 From 5629aade36916b967ffae250812e833816e0dddd Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 29 Jun 2012 06:18:42 +0000 Subject: [PATCH 144/155] Upgrade sqlalchemy to 0.7.8 (latest) --- pip-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pip-requirements.txt b/pip-requirements.txt index 5db1fc4c09a..2bb53343f1a 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -4,7 +4,7 @@ # instructions in CKAN's documentation for full installation instructions. Genshi==0.6 sqlalchemy-migrate==0.7.2 -sqlalchemy==0.7.3 +sqlalchemy==0.7.8 webhelpers==1.3 pyutilib.component.core==4.5.3 -e git+https://github.com/okfn/vdm.git@vdm-0.11#egg=vdm From cd7974b2be434d88d937f1fbad2042413b463c48 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 14:02:30 +0200 Subject: [PATCH 145/155] Refactor form_to_db_schema_options Make it call self.form_to_db_schema so that the form_to_db_schema() methods of IDatasetForm extensions get called. Also make it call new form_to_db_schema_api_create() and form_to_db_schema_api_update() methods which could potentially be overridden by extensions also. --- ckan/lib/plugins.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ckan/lib/plugins.py b/ckan/lib/plugins.py index 258e1449681..4ce1f4236ca 100644 --- a/ckan/lib/plugins.py +++ b/ckan/lib/plugins.py @@ -221,11 +221,21 @@ def form_to_db_schema_options(self, options): if options.get('api'): if options.get('type') == 'create': - return logic.schema.default_create_package_schema() + return self.form_to_db_schema_api_create() else: - return logic.schema.default_update_package_schema() + assert options.get('type') == 'update' + return self.form_to_db_schema_api_update() else: - return logic.schema.package_form_schema() + return self.form_to_db_schema() + + def form_to_db_schema(self, options): + return logic.schema.package_form_schema() + + def form_to_db_schema_api_create(self): + return logic.schema.default_create_package_schema() + + def form_to_db_schema_api_update(self): + return logic.schema.default_update_package_schema() def db_to_form_schema(self): '''This is an interface to manipulate data from the database From 90012807433047593f7acac22522af5bc1302ac6 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 14:32:00 +0200 Subject: [PATCH 146/155] Fix a crasher in package/read.html template Sometimes c.pkg_dict has no member named groups (e.g. when there is an active IDatasetForm plugin with a db_to_form_schema() method). --- ckan/templates/package/read.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/package/read.html b/ckan/templates/package/read.html index 29a03720bbd..7c76ef39192 100644 --- a/ckan/templates/package/read.html +++ b/ckan/templates/package/read.html @@ -45,7 +45,7 @@

Tags

${tag_list(c.pkg_dict.get('tags', ''))} -
  • +

  • Groups

    • From a5bedce1d321752fd083a7a4ec10e263439e713d Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 14:51:11 +0200 Subject: [PATCH 147/155] Add db_to_form_schema method For IDatasetForm plugins to base their db_to_form_schema() dicts on --- ckan/logic/schema.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 8257408bb68..301f46b0b8b 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -175,6 +175,23 @@ def package_form_schema(): schema.pop('relationships_as_subject') return schema +def db_to_form_schema(): + schema = default_package_schema() + # Workaround a bug in CKAN's convert_from_tags() function. + # TODO: Fix this issue in convert_from_tags(). + schema.update({ + 'tags': { + '__extras': [ckan.lib.navl.validators.keep_extras, + ckan.logic.converters.free_tags_only] + }, + }) + # Workaround a bug in CKAN. + # TODO: Fix this elsewhere so we don't need to workaround it here. + schema['resources'].update({ + 'created': [ckan.lib.navl.validators.ignore_missing], + }) + return schema + def default_group_schema(): schema = { From 7f59f2539abfa009618d34deea087a97a4a94dba Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 15:02:05 +0200 Subject: [PATCH 148/155] Rename db_to_form_schema -> db_to_form_package_schema() --- ckan/logic/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 301f46b0b8b..5ae86675633 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -175,7 +175,7 @@ def package_form_schema(): schema.pop('relationships_as_subject') return schema -def db_to_form_schema(): +def db_to_form_package_schema(): schema = default_package_schema() # Workaround a bug in CKAN's convert_from_tags() function. # TODO: Fix this issue in convert_from_tags(). From 38fc15cfc0965e567444b2653ff5a36b3f33828b Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 15:20:14 +0200 Subject: [PATCH 149/155] Rename package_form_schema() -> form_to_db_package_schema() The only purpose of this schema is to be used as a base schema by IDatasetForm plugins for their form_to_db_schema() methods, hopefully the new name makes that clearer --- ckan/lib/plugins.py | 6 +++--- ckan/logic/schema.py | 2 +- ckan/tests/schema/test_schema.py | 2 +- ckanext/publisher_form/forms.py | 1 - ckanext/test_tag_vocab_plugin.py | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ckan/lib/plugins.py b/ckan/lib/plugins.py index 4ce1f4236ca..bb5b795aa18 100644 --- a/ckan/lib/plugins.py +++ b/ckan/lib/plugins.py @@ -228,8 +228,8 @@ def form_to_db_schema_options(self, options): else: return self.form_to_db_schema() - def form_to_db_schema(self, options): - return logic.schema.package_form_schema() + def form_to_db_schema(self): + return logic.schema.form_to_db_package_schema() def form_to_db_schema_api_create(self): return logic.schema.default_create_package_schema() @@ -397,7 +397,7 @@ def check_data_dict(self, data_dict): 'extras_validation', 'save', 'return_to', 'resources'] - schema_keys = package_form_schema().keys() + schema_keys = form_to_db_package_schema().keys() keys_in_schema = set(schema_keys) - set(surplus_keys_schema) missing_keys = keys_in_schema - set(data_dict.keys()) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 5ae86675633..ba4f9472afb 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -152,7 +152,7 @@ def default_update_package_schema(): return schema -def package_form_schema(): +def form_to_db_package_schema(): schema = default_package_schema() ##new diff --git a/ckan/tests/schema/test_schema.py b/ckan/tests/schema/test_schema.py index e8d67e2c547..ab2c3d629d6 100644 --- a/ckan/tests/schema/test_schema.py +++ b/ckan/tests/schema/test_schema.py @@ -94,7 +94,7 @@ def test_tag_string_parsing(self): # errors correctly. context = {'model': ckan.model, 'session': ckan.model.Session} - schema = ckan.logic.schema.package_form_schema() + schema = ckan.logic.schema.form_to_db_package_schema() # basic parsing of comma separated values tests = (('tag', ['tag'], []), diff --git a/ckanext/publisher_form/forms.py b/ckanext/publisher_form/forms.py index 672209f6b4a..0d23d7aed10 100644 --- a/ckanext/publisher_form/forms.py +++ b/ckanext/publisher_form/forms.py @@ -9,7 +9,6 @@ from ckan.logic import tuplize_dict, clean_dict, parse_params import ckan.logic.schema as default_schema from ckan.logic.schema import group_form_schema -from ckan.logic.schema import package_form_schema import ckan.logic.validators as val from ckan.lib.base import BaseController, render, c, model, abort, request from ckan.lib.base import redirect, _, config, h diff --git a/ckanext/test_tag_vocab_plugin.py b/ckanext/test_tag_vocab_plugin.py index 043234890b2..2fe95f4ed9d 100644 --- a/ckanext/test_tag_vocab_plugin.py +++ b/ckanext/test_tag_vocab_plugin.py @@ -7,7 +7,7 @@ from genshi.filters import Transformer from ckan.logic import get_action from ckan.logic.converters import convert_to_tags, convert_from_tags, free_tags_only -from ckan.logic.schema import package_form_schema, default_package_schema +from ckan.logic.schema import default_package_schema from ckan.lib.navl.validators import ignore_missing, keep_extras from ckan import plugins From c5553b8e0fdd3e0394c97880f343bee4615b2301 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 15:32:51 +0200 Subject: [PATCH 150/155] Add back in deprectated package_form_schema() for backwards compatibility --- ckan/logic/schema.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index ba4f9472afb..2bb0b58b3ac 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -152,6 +152,11 @@ def default_update_package_schema(): return schema +def package_form_schema(): + # This function is deprecated and was replaced by + # form_to_db_package_schema(), it remains here for backwards compatibility. + return form_to_db_package_schema() + def form_to_db_package_schema(): schema = default_package_schema() From efca8cb18ca6379a51cb81e59d347ecb370ec78c Mon Sep 17 00:00:00 2001 From: kindly Date: Fri, 6 Jul 2012 15:01:45 +0100 Subject: [PATCH 151/155] [2627] fix some more date issues when displayed --- ckan/logic/schema.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 2bb0b58b3ac..f3b27654b74 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -194,6 +194,9 @@ def db_to_form_package_schema(): # TODO: Fix this elsewhere so we don't need to workaround it here. schema['resources'].update({ 'created': [ckan.lib.navl.validators.ignore_missing], + 'last_modified': [ckan.lib.navl.validators.ignore_missing], + 'cache_last_updated': [ckan.lib.navl.validators.ignore_missing], + 'webstore_last_updated': [ckan.lib.navl.validators.ignore_missing], }) return schema From 519472c1bcf15f66e18927f775e06e5ee70724fa Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Sat, 7 Jul 2012 12:21:55 +0100 Subject: [PATCH 152/155] [#2437] Add config file argument to paster commands. --- doc/coding-standards.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/coding-standards.rst b/doc/coding-standards.rst index 5f575c174b0..9ed4e291b04 100644 --- a/doc/coding-standards.rst +++ b/doc/coding-standards.rst @@ -389,7 +389,7 @@ CKAN currently uses to manage these scripts. When you deploy new code to a CKAN instance, as part of the process you run any required migration scripts with: :: - paster db upgrade + paster --plugin=ckan db upgrade --config={.ini file} The scripts give their model version numbers in their filenames and are stored in ``ckan/migration/versions/``. @@ -428,14 +428,16 @@ made: 1. Get a dump of the database schema before you add your new migrate scripts. :: - paster db clean - paster db upgrade + paster --plugin=ckan db clean --config={.ini file} + paster --plugin=ckan db upgrade --config={.ini file} pg_dump -h host -s -f old.sql dbname 2. Get a dump of the database as you have specified it in the model. :: - paster db clean - paster db create-from-model #this makes the database as defined in the model + paster --plugin=ckan db clean --config={.ini file} + + #this makes the database as defined in the model + paster --plugin=ckan db create-from-model -config={.ini file} pg_dump -h host -s -f new.sql dbname 3. Get agpdiff (apt-get it). It produces sql it thinks that you need to run on From 29a06dc503681cb0ab7951ea05cf55b47d025998 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Sat, 7 Jul 2012 12:28:25 +0100 Subject: [PATCH 153/155] [#2437] Update commit message guidelines --- doc/coding-standards.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/coding-standards.rst b/doc/coding-standards.rst index 9ed4e291b04..32987bc6150 100644 --- a/doc/coding-standards.rst +++ b/doc/coding-standards.rst @@ -66,9 +66,10 @@ Feature Branches ---------------- All ticketed work should be developed on a corresponding feature branch forked -from master. The name of the branch should inlude the `type` of the ticket, -the ticket's number, and a brief one-line synopsis of the purpose of the -ticket. eg: ``enhancement-2298-add-sort-by-controls-to-search-page``. +from master. The name of the branch should inlude the ticket's number, the +ticket type, and a brief one-line synopsis of the purpose of the ticket. eg: +``2298-feature-add-sort-by-controls-to-search-page``. This allows the ticket +number to be esaily searchable through github's web interface. Once work on the branch has been completed and it is ready to be merged into master, make a pull request on github. Another member of the CKAN team will From 3a518547e18a1b9d2177db9243620272c58f3ba9 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Mon, 9 Jul 2012 09:43:38 +0100 Subject: [PATCH 154/155] Fixes 2624, where tags were not saved with organization_dataset Added tag_convert_string to the validators for the form_to_db_schema so that it correctly converted the string to tags in the DB. --- ckanext/organizations/forms.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckanext/organizations/forms.py b/ckanext/organizations/forms.py index caf9226c2da..3e22ff4e297 100644 --- a/ckanext/organizations/forms.py +++ b/ckanext/organizations/forms.py @@ -11,7 +11,7 @@ from ckan.plugins import IGroupForm, IDatasetForm, IConfigurer, IRoutes from ckan.plugins import implements, SingletonPlugin from ckan.logic import check_access - +from ckan.logic.validators import tag_string_convert from ckan.lib.navl.validators import (ignore_missing, not_empty, empty, @@ -237,6 +237,7 @@ def form_to_db_schema(self): schema = default_package_schema() schema['groups']['capacity'] = [ignore_missing, unicode] schema['__after'] = [group_required] + schema['tag_string'] = [ignore_missing, tag_string_convert] return schema def check_data_dict(self, data_dict, schema=None): From 4f32c46e50d2c9767faf6768ac013248d83d8d82 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 9 Jul 2012 12:37:32 +0200 Subject: [PATCH 155/155] [#2624] Base organizations form_to_db_schema() off form_to_db_package_schema() --- ckanext/organizations/forms.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ckanext/organizations/forms.py b/ckanext/organizations/forms.py index 3e22ff4e297..5e33eaa8f24 100644 --- a/ckanext/organizations/forms.py +++ b/ckanext/organizations/forms.py @@ -3,7 +3,7 @@ import ckan.authz as authz from ckan.logic import NotAuthorized -from ckan.logic.schema import group_form_schema, default_package_schema +from ckan.logic.schema import group_form_schema, form_to_db_package_schema from ckan.lib import base from ckan.lib.base import c, model, abort, request from ckan.lib.base import redirect, _, config, h @@ -234,10 +234,9 @@ def db_to_form_schema(self): into a format suitable for the form (optional)''' def form_to_db_schema(self): - schema = default_package_schema() + schema = form_to_db_package_schema() schema['groups']['capacity'] = [ignore_missing, unicode] schema['__after'] = [group_required] - schema['tag_string'] = [ignore_missing, tag_string_convert] return schema def check_data_dict(self, data_dict, schema=None):