From b6b981c07744c1193f82a64c08087d6d934c7d8a Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 23 Jan 2014 09:53:37 +0000 Subject: [PATCH 1/6] [#1446] Require ckan.site_url when using the DataPusher --- ckanext/datapusher/plugin.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py index 50c9cd864f5..f0b03936359 100644 --- a/ckanext/datapusher/plugin.py +++ b/ckanext/datapusher/plugin.py @@ -80,10 +80,11 @@ def configure(self, config): datapusher_formats = config.get('ckan.datapusher.formats', '').lower() self.datapusher_formats = datapusher_formats.split() or DEFAULT_FORMATS - datapusher_url = config.get('ckan.datapusher.url') - if not datapusher_url: - raise Exception( - 'Config option `ckan.datapusher.url` has to be set.') + for config_option in ('ckan.site_url', 'ckan.datapusher.url',): + if not config.get(config_option): + raise Exception( + 'Config option `{0}` must be set to use the DataPusher.' + .format(config_option)) def notify(self, entity, operation=None): if isinstance(entity, model.Resource): From 0230587df78bae50288272f78b0a69527335e1fa Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 23 Jan 2014 11:25:55 +0000 Subject: [PATCH 2/6] [#1446] Better defaults for DataPusher config options + docs --- ckan/config/deployment.ini_tmpl | 4 ++-- doc/configuration.rst | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index b3148cc6d68..a8c05cb92f6 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -130,8 +130,8 @@ ckan.feeds.author_link = # Make sure you have set up the DataStore -ckan.datapusher.formats = csv -ckan.datapusher.url = http://datapusher.ckan.org/ +#ckan.datapusher.formats = +#ckan.datapusher.url = http://127.0.0.1:8800/ ## Activity Streams Settings diff --git a/doc/configuration.rst b/doc/configuration.rst index a9c7a581cc7..cd05f2b777e 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1066,9 +1066,15 @@ ckan.datapusher.formats ^^^^^^^^^^^^^^^^^^^^^^^ Example:: - ckan.datapusher.formats = csv xls xlsx -.. todo:: Expand + ckan.datapusher.formats = csv xls + +Default value: ``csv xls application/csv application/vnd.ms-excel`` + +File formats that will be pushed to the DataStore by the DataPusher. When +adding or editing a resource which links to a file in one of these formats, +the DataPusher will automatically try to import its contents to the DataStore. + .. _ckan.datapusher.url: @@ -1076,9 +1082,13 @@ ckan.datapusher.url ^^^^^^^^^^^^^^^^^^^ Example:: - ckan.datapusher.url = http://datapusher.ckan.org/ -.. todo:: Expand + ckan.datapusher.url = http://127.0.0.1:8800/ + +DataPusher endpoint to use when enabling the ``datapusher`` extension. If you +installed CKAN via :doc:`install-from-package`, the DataPusher was installed for you +running on port 8800. If you want to manually install the DataPusher, follow +the installation `instructions `_. Activity Streams Settings From b128ebb1437c4f882be1304feb2020eebddc95a8 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 23 Jan 2014 11:52:08 +0000 Subject: [PATCH 3/6] [#1446] Clarify DataPusher issues in docs --- doc/datastore.rst | 25 +++++++++++++++++++++---- doc/install-from-package.rst | 9 +++++++-- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/doc/datastore.rst b/doc/datastore.rst index 80fe5955bc3..026c1f303fd 100644 --- a/doc/datastore.rst +++ b/doc/datastore.rst @@ -15,6 +15,11 @@ When a resource is added to the DataStore, you get: The DataStore is integrated into the :doc:`CKAN API ` and authorization system. +The DataStore is generally used alongside the +`DataPusher `_, which will +automatically upload data to the DataStore from suitable files, whether +uploaded to CKAN's FileStore or externally linked. + .. contents:: :depth: 1 :local: @@ -126,7 +131,16 @@ Option 1: Paster command This option is preferred if CKAN and PostgreSQL are on the same server. -To set the permissions, use this paster command after you've set the database URLs (make sure to have your virtualenv activated): +To set the permissions, use the following paster command after you've set the database URLs. + +If you did a package install, the easiest way is to use the ``ckan`` command wrapper: + +.. parsed-literal:: + + sudo ckan datastore set-permissions postgres + +If you did a source install, make sure to have your virtualenv activated and +run the command from the CKAN source directory: .. parsed-literal:: @@ -230,10 +244,13 @@ DataStore. This requires some processing, to extract the data from your files and to add it to the DataStore in the format the DataStore can handle. This task of automatically parsing and then adding data to the DataStore is -performed by a DataPusher, a service that runs asynchronously and can be installed -allongside CKAN. +performed by the `DataPusher `_, a service that runs asynchronously and can be installed +alongside CKAN. + +To install this please look at the docs here: http://docs.ckan.org/projects/datapusher + -To install this please look at the docs here: http://datapusher.readthedocs.org +.. _DataPusher_docs: http://docs.ckan.org/projects/datapusher ----------------- diff --git a/doc/install-from-package.rst b/doc/install-from-package.rst index f42bb92e30f..690ad7577c2 100644 --- a/doc/install-from-package.rst +++ b/doc/install-from-package.rst @@ -9,6 +9,11 @@ and easiest way to install CKAN, but it requires **Ubuntu 12.04 64-bit**. If you're not using Ubuntu 12.04 64-bit, or if you're installing CKAN for development, you should follow :doc:`install-from-source` instead. +At the end of the installation process you will end up with two running web +applications, CKAN itself and the DataPusher, a separate service for automatically +importing data to CKAN's :doc:`datastore`. + + .. _run-package-installer: --------------------------- @@ -86,8 +91,8 @@ CKAN: sudo ckan db init -#. Optionally, setup the DataStore by following the instructions in - :doc:`/datastore`. +#. Optionally, setup the DataStore and DataPusher by following the + instructions in :doc:`/datastore`. #. Also optionally, you can enable file uploads by following the instructions in :doc:`filestore`. From 853568a023662308113fb1756d606f1c0c4b216c Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 23 Jan 2014 11:54:12 +0000 Subject: [PATCH 4/6] [#1446] Remove SQLite urls from template ini file --- ckan/config/deployment.ini_tmpl | 2 -- 1 file changed, 2 deletions(-) diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index a8c05cb92f6..b15d1c5657b 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -44,8 +44,6 @@ who.log_file = %(cache_dir)s/who_log.ini ## Database Settings sqlalchemy.url = postgresql://ckan_default:pass@localhost/ckan_default -#sqlalchemy.url = sqlite:/// -#sqlalchemy.url = sqlite:///%(here)s/somedb.db #ckan.datastore.write_url = postgresql://ckan_default:pass@localhost/datastore_default #ckan.datastore.read_url = postgresql://datastore_default:pass@localhost/datastore_default From 1613ed86d45a7694281ba9b161cb5a2c23dc8acf Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 23 Jan 2014 16:50:38 +0000 Subject: [PATCH 5/6] [#1446] Enable DataPusher for private datasets There's no need to prevent the import to the datastore on them, as the datastore will manage the authorization, and logged in users still expect these files to get imported to the DataStore. --- ckanext/datapusher/plugin.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py index f0b03936359..56fb44d2122 100644 --- a/ckanext/datapusher/plugin.py +++ b/ckanext/datapusher/plugin.py @@ -95,10 +95,7 @@ def notify(self, entity, operation=None): # 1 parameter context = {'model': model, 'ignore_auth': True, 'defer_commit': True} - package = p.toolkit.get_action('package_show')(context, { - 'id': entity.get_package_id() - }) - if (not package['private'] and entity.format and + if (entity.format and entity.format.lower() in self.datapusher_formats and entity.url_type != 'datapusher'): try: From 699dac215996c01afecc93fc062d3a230d8e810e Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 23 Jan 2014 16:53:00 +0000 Subject: [PATCH 6/6] [#1446] Remove misleading datastore warning in docs --- doc/datastore.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/datastore.rst b/doc/datastore.rst index 026c1f303fd..41519aa6ed5 100644 --- a/doc/datastore.rst +++ b/doc/datastore.rst @@ -51,10 +51,6 @@ Setting up the DataStore available and the set-up is slightly different. Make sure, you read :ref:`legacy-mode` for more details. -.. warning:: - - The DataStore does not support hiding resources in a private dataset. - 1. Enable the plugin ====================