From 7accd54887a5c6767c503e30ea616f669f044e6f Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 25 Jul 2012 19:42:10 +0100 Subject: [PATCH 01/10] [#2757] Add source install upgrade docs --- doc/install-from-source.rst | 73 +++++++++++++++++++++++++++++++++++++ doc/paster.rst | 5 +++ doc/solr-setup.rst | 2 + 3 files changed, 80 insertions(+) diff --git a/doc/install-from-source.rst b/doc/install-from-source.rst index 64bdc79a3a5..5b1fbe0ca30 100644 --- a/doc/install-from-source.rst +++ b/doc/install-from-source.rst @@ -221,3 +221,76 @@ they all pass. See :doc:`test`. You can now proceed to :doc:`post-installation` which covers creating a CKAN sysadmin account and deploying CKAN with Apache. + +Upgrading a source install +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Before upgrading your version of CKAN you should check that any custom +templates or extensions you're using work with the new version of CKAN. For +example, you could install the new version of CKAN in a new virtual environment +and use that to test your templates and extensions. + +You should also read the `CKAN Changelog `_ +to see if there are any extra notes to be aware of when upgrading to the new +version. + +1. Backup your CKAN database using the ``ckan db dump`` command, for example:: + + paster --plugin=ckan db dump --config=/path/to/your/ckan.ini my_ckan_database.pg_dump + + This will create a file called ``my_ckan_database.pg_dump``, if something + goes wrong with the CKAN upgrade you can use this file to restore the + database to its pre-upgrade state. See :ref:`dumping and loading` for + details of the `ckan db dump` and `ckan db load` commands. + +2. Checkout the new CKAN version from git, for example:: + + cd pyenv/src/ckan + git fetch + git checkout release-v1.8.1 + + If you have any CKAN extensions installed from source, you may need to + checkout newer versions of the extensions at this point as well. Refer to + the documentation for each extension. + +3. Update CKAN's dependencies. Make sure that your CKAN virtual environment + is active, then run this command:: + + pip install --upgrade -r /path/to/your/pyenv/ckan/ckan/pip-requirements.txt + +4. If you are upgrading to a new major version of CKAN (for example if you are + upgrading to CKAN 1.7, 1.8 or 1.9, etc.), update your CKAN database's schema + using the ``ckan db upgrade`` command, for example:: + + paster --plugin=ckan db upgrade --config=/path/to/your/ckan.ini + + If you are just upgrading to a minor version of CKAN (for example upgrading + from version 1.8 to 1.8.1) then it should not be necessary to upgrade your + database. + + See :ref:`upgrade migration` for details of the ``ckan db upgrade`` command. + +5. If CKAN's Solr schema version has changed between the CKAN versions you're + upgrading from and to, then you need to update your solr schema symlink. + + When :ref:`setting up solr` you created a symlink + ``/etc/solr/conf/schema.xml`` linking to a CKAN Solr schema file such as + ``/path/to/your/pyenv/ckan/ckan/config/solr/schema-1.4.xml``. This symlink + should be updated to point to the latest schema file in + ``/path/to/your/pyenv/ckan/ckan/config/solr/``, if it doesn't already. + + After updating the symlink, you must rebuild your search index by running + the ``ckan search-index rebuild`` command, for example:: + + paster --plugin=ckan search-index rebuild --config=/path/to/your/ckan.ini + + See :ref:`rebuild search index` for details of the + ``ckan search-index rebuild`` command. + +6. Finally, restart your web server. For example if you have deployed CKAN + using the Apache web server on Ubuntu linux, run this command:: + + sudo service apache2 restart + +7. You're done! You should now be able to visit your CKAN website in your web + browser and see that it's now running the new version of CKAN. diff --git a/doc/paster.rst b/doc/paster.rst index bf94e9f6b21..59350aa6b3f 100644 --- a/doc/paster.rst +++ b/doc/paster.rst @@ -184,6 +184,8 @@ You can delete everything in the CKAN database, including the tables, to start f The next logical step from this point is to do a "db init" step before starting CKAN again. +.. _dumping and loading: + Dumping and Loading databases to/from a file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -202,6 +204,8 @@ To load it in again, you first have to clean the database of existing data (be c The pg_dump file notes which PostgreSQL user 'owns' the data on the server. Because the PostgreSQL user (by default) is identified as the current Linux user, and this is setup to be ``ckanINSTANCE`` where ``INSTANCE`` is the name of the CKAN instance. This means if you want to restore the pg_dump as another CKAN instance name (often needed if you move it to another server) then you will need to change the database owner - see :doc:`howto-editing-database-ownership`. +.. _upgrade migration: + Upgrade migration ~~~~~~~~~~~~~~~~~ @@ -248,6 +252,7 @@ This important command gives you fine-grained control over CKAN permissions, by The ``roles`` command has its own section: see :doc:`authorization`. +.. _rebuild search index: search-index: Rebuild search index ---------------------------------- diff --git a/doc/solr-setup.rst b/doc/solr-setup.rst index a280a69cb0c..62efe6fef16 100644 --- a/doc/solr-setup.rst +++ b/doc/solr-setup.rst @@ -1,3 +1,5 @@ +.. _setting up solr: + =============== Setting up Solr =============== From 0bd3db4ba5919f4a88439862375177cf93d83ab8 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 26 Jul 2012 13:27:57 +0100 Subject: [PATCH 02/10] [#2757] Add some comments to source upgrade --- doc/install-from-source.rst | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/doc/install-from-source.rst b/doc/install-from-source.rst index 5b1fbe0ca30..331d04f5431 100644 --- a/doc/install-from-source.rst +++ b/doc/install-from-source.rst @@ -260,7 +260,15 @@ version. 4. If you are upgrading to a new major version of CKAN (for example if you are upgrading to CKAN 1.7, 1.8 or 1.9, etc.), update your CKAN database's schema - using the ``ckan db upgrade`` command, for example:: + using the ``ckan db upgrade`` command. + + .. warning :: + + To avoid problems during the database upgrade, comment out any + plugins that you have enabled on your ini file. You can uncomment + them back when the upgrade finishes. + + For example:: paster --plugin=ckan db upgrade --config=/path/to/your/ckan.ini @@ -271,7 +279,9 @@ version. See :ref:`upgrade migration` for details of the ``ckan db upgrade`` command. 5. If CKAN's Solr schema version has changed between the CKAN versions you're - upgrading from and to, then you need to update your solr schema symlink. + upgrading from and to, then you need to update your solr schema symlink + (Check the CHANGELOG to see if it necessary to update the schema, otherwise + you can skip this step). When :ref:`setting up solr` you created a symlink ``/etc/solr/conf/schema.xml`` linking to a CKAN Solr schema file such as From cfdeb919f228f917b287eda203e5a5371e0977bc Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Mon, 23 Jul 2012 16:02:25 +0100 Subject: [PATCH 03/10] Fix for relative url being used when doing file upload to local storage --- ckan/controllers/storage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ckan/controllers/storage.py b/ckan/controllers/storage.py index 3f605117650..c60eb6148d6 100644 --- a/ckan/controllers/storage.py +++ b/ckan/controllers/storage.py @@ -269,6 +269,9 @@ def get_metadata(self, label): label=label, qualified=False ) + if url.startswith('/'): + url = config.get('ckan.site_url','') + url + if not self.ofs.exists(bucket, label): abort(404) metadata = self.ofs.get_metadata(bucket, label) From 0bdaf23147f518e45a10bb181b83507ab45f45b1 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 27 Jul 2012 12:39:02 +0100 Subject: [PATCH 04/10] [#2770] Mark strings for translation on add related item form --- ckan/templates/related/add-related.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ckan/templates/related/add-related.html b/ckan/templates/related/add-related.html index 1e5f23e49d5..a84793bbb98 100644 --- a/ckan/templates/related/add-related.html +++ b/ckan/templates/related/add-related.html @@ -16,7 +16,7 @@

Add item

- +
@@ -32,15 +32,15 @@

Add item

- +
- +
- +
From 9acc9ea27723ed4ee90380512652f7c37114b18f Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 30 Jul 2012 20:23:47 +0100 Subject: [PATCH 05/10] [i18n] Mark 2 resource form strings for i18n --- ckan/public/scripts/templates.js | 4 ++-- ckan/templates/js_strings.html | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ckan/public/scripts/templates.js b/ckan/public/scripts/templates.js index 596c94f45fc..8256567345c 100644 --- a/ckan/public/scripts/templates.js +++ b/ckan/public/scripts/templates.js @@ -152,7 +152,7 @@ CKAN.Templates.resourceDetails = ' \ CKAN.Templates.resourceExtra = ' \
\ \ - \ - \ + \ + \
\ '; diff --git a/ckan/templates/js_strings.html b/ckan/templates/js_strings.html index bcce855880f..96060be00bc 100644 --- a/ckan/templates/js_strings.html +++ b/ckan/templates/js_strings.html @@ -63,6 +63,8 @@ unknown = _('Unknown'), extraFields = _('Extra Fields'), addExtraField = _('Add Extra Field'), + key = _('Key'), + value = _('Value'), deleteResource = _('Delete Resource'), youCanUseMarkdown = _('You can use %aMarkdown formatting%b here.'), datesAreInISO = _('Dates are in %aISO Format%b — eg. %c2012-12-25%d or %c2010-05-31T14:30%d.'), From 613322a3cbb0041815fb9118b372cd9404c2e1a4 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 31 Jul 2012 18:13:59 +0100 Subject: [PATCH 06/10] [#2777] Use user_list_dictize --- ckan/logic/action/get.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index cc7d14e037d..f74843e835b 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -2049,7 +2049,7 @@ def _follower_list(context, data_dict, FollowerClass): users = [user for user in users if user is not None] # Dictize the list of User objects. - return [model_dictize.user_dictize(user,context) for user in users] + return model_dictize.user_list_dictize(users, context) def user_follower_list(context, data_dict): '''Return the list of users that are following the given user. @@ -2192,7 +2192,7 @@ def user_followee_list(context, data_dict): users = [user for user in users if user is not None] # Dictize the list of User objects. - return [model_dictize.user_dictize(user, context) for user in users] + return model_dictize.user_list_dictize(users, context) def dataset_followee_list(context, data_dict): '''Return the list of datasets that are followed by the given user. From 6a48063b971a5f6813c5034eace85e479b5308df Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Sat, 28 Jul 2012 19:32:44 +0100 Subject: [PATCH 07/10] [#2750] Fix group controller using wrong schema Fix group controller calling form_to_db_schema() from IGroupForm plugin when it should be using db_to_form_schema() --- ckan/controllers/group.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index f0e42a839de..1b7557e681b 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -34,7 +34,7 @@ def _form_to_db_schema(self, group_type=None): def _db_to_form_schema(self, group_type=None): '''This is an interface to manipulate data from the database into a format suitable for the form (optional)''' - return lookup_group_plugin(group_type).form_to_db_schema() + return lookup_group_plugin(group_type).db_to_form_schema() def _setup_template_variables(self, context, data_dict, group_type=None): return lookup_group_plugin(group_type).\ @@ -100,7 +100,7 @@ def read(self, id): group_type = self._get_group_type(id.split('@')[0]) context = {'model': model, 'session': model.Session, 'user': c.user or c.author, - 'schema': self._form_to_db_schema(group_type=group_type), + 'schema': self._db_to_form_schema(group_type=group_type), 'for_view': True, 'extras_as_string': True} data_dict = {'id': id} # unicode format (decoded from utf8) From 3ed510b2da164e566fc2c23eee84a495f6800cb3 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 19 Jul 2012 19:48:39 +0200 Subject: [PATCH 08/10] Refactor DefaultGroupForm's form_to_db_schema_options() The form_to_db_schema() methods of IGroupForm plugins were not being called if those plugins inherited from DefaultGroupForm. Refactor it in the same way as DefaultDatasetForm's form_to_db_schema_options() method was recently refactored. Make it call self.form_to_db_schema so that the form_to_db_schema() methods of IGroupForm extensions get called. Also make it call new form_to_db_schema_api_create() and form_to_db_schema_api_update() methods which could potentially be overridden by extensions also. --- ckan/lib/plugins.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/ckan/lib/plugins.py b/ckan/lib/plugins.py index 2fccd93ec03..20910fb6aeb 100644 --- a/ckan/lib/plugins.py +++ b/ckan/lib/plugins.py @@ -349,9 +349,6 @@ def history_template(self): def group_form(self): return 'group/new_group_form.html' - def form_to_db_schema(self): - return logic.schema.group_form_schema() - def form_to_db_schema_options(self, options): ''' This allows us to select different schemas for different purpose eg via the web interface or via the api or creation vs @@ -366,11 +363,20 @@ def form_to_db_schema_options(self, options): if options.get('api'): if options.get('type') == 'create': - return logic.schema.default_group_schema() + return self.form_to_db_schema_api_create() else: - return logic.schema.default_update_group_schema() + return self.form_to_db_schema_api_update() else: - return logic.schema.group_form_schema() + return self.form_to_db_schema() + + def form_to_db_schema_api_create(self): + return logic.schema.default_group_schema() + + def form_to_db_schema_api_update(self): + return logic.schema.default_update_group_schema() + + def form_to_db_schema(self): + return logic.schema.group_form_schema() def db_to_form_schema(self): '''This is an interface to manipulate data from the database From 5c63f301ae22f6f54a62afa0d0211be3cee628cb Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Sat, 28 Jul 2012 19:41:25 +0100 Subject: [PATCH 09/10] [#2570] Fix IGroupForm check_data_dict not being called Make group_create() and group_update() call the check_data_dict() methods of IGroupForm plugins, like package_create() and package_update() do with IDatasetForm plugins. --- ckan/logic/action/create.py | 8 ++++++++ ckan/logic/action/update.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 507acbb7d3f..94b5f74ce32 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -512,6 +512,14 @@ def group_create(context, data_dict): except AttributeError: schema = group_plugin.form_to_db_schema() + if 'api_version' not in context: + # old plugins do not support passing the schema so we need + # to ensure they still work + try: + group_plugin.check_data_dict(data_dict, schema) + except TypeError: + group_plugin.check_data_dict(data_dict) + data, errors = _validate(data_dict, schema, context) log.debug('group_create validate_errs=%r user=%s group=%s data_dict=%r', errors, context.get('user'), data_dict.get('name'), data_dict) diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index fa5d8275e22..a06274c0714 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -415,6 +415,14 @@ def group_update(context, data_dict): _check_access('group_update', context, data_dict) + if 'api_version' not in context: + # old plugins do not support passing the schema so we need + # to ensure they still work + try: + group_plugin.check_data_dict(data_dict, schema) + except TypeError: + group_plugin.check_data_dict(data_dict) + data, errors = _validate(data_dict, schema, context) log.debug('group_update validate_errs=%r user=%s group=%s data_dict=%r', errors, context.get('user'), From e2073a37ac16acb1233a19f42d6473e0a2065b75 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 1 Aug 2012 13:42:19 +0100 Subject: [PATCH 10/10] Be more careful when prepending site_url --- ckan/controllers/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/controllers/storage.py b/ckan/controllers/storage.py index c60eb6148d6..19dd64554a1 100644 --- a/ckan/controllers/storage.py +++ b/ckan/controllers/storage.py @@ -270,7 +270,7 @@ def get_metadata(self, label): qualified=False ) if url.startswith('/'): - url = config.get('ckan.site_url','') + url + url = config.get('ckan.site_url','').rstrip('/') + '/' + url if not self.ofs.exists(bucket, label): abort(404)