Merge pull request #1446 from okfn/1446-datapusher-integration-issues

DataPusher integration issues

ckan/config/deployment.ini_tmpl

@@ -44,8 +44,6 @@ who.log_file = %(cache_dir)s/who_log.ini
 
 ## Database Settings
 sqlalchemy.url = postgresql://ckan_default:pass@localhost/ckan_default
-#sqlalchemy.url = sqlite:///
-#sqlalchemy.url = sqlite:///%(here)s/somedb.db
 
 #ckan.datastore.write_url = postgresql://ckan_default:pass@localhost/datastore_default
 #ckan.datastore.read_url = postgresql://datastore_default:pass@localhost/datastore_default
@@ -130,8 +128,8 @@ ckan.feeds.author_link =
 
 # Make sure you have set up the DataStore
 
-ckan.datapusher.formats = csv
-ckan.datapusher.url = http://datapusher.ckan.org/
+#ckan.datapusher.formats =
+#ckan.datapusher.url = http://127.0.0.1:8800/
 
 ## Activity Streams Settings
 

ckanext/datapusher/plugin.py

@@ -80,10 +80,11 @@ def configure(self, config):
         datapusher_formats = config.get('ckan.datapusher.formats', '').lower()
         self.datapusher_formats = datapusher_formats.split() or DEFAULT_FORMATS
 
-        datapusher_url = config.get('ckan.datapusher.url')
-        if not datapusher_url:
-            raise Exception(
-                'Config option `ckan.datapusher.url` has to be set.')
+        for config_option in ('ckan.site_url', 'ckan.datapusher.url',):
+            if not config.get(config_option):
+                raise Exception(
+                    'Config option `{0}` must be set to use the DataPusher.'
+                    .format(config_option))
 
     def notify(self, entity, operation=None):
         if isinstance(entity, model.Resource):
@@ -94,10 +95,7 @@ def notify(self, entity, operation=None):
                 # 1 parameter
                 context = {'model': model, 'ignore_auth': True,
                            'defer_commit': True}
-                package = p.toolkit.get_action('package_show')(context, {
-                    'id': entity.get_package_id()
-                })
-                if (not package['private'] and entity.format and
+                if (entity.format and
                         entity.format.lower() in self.datapusher_formats and
                         entity.url_type != 'datapusher'):
                     try:

doc/configuration.rst

@@ -1066,19 +1066,29 @@ ckan.datapusher.formats
 ^^^^^^^^^^^^^^^^^^^^^^^
 
 Example::
-  ckan.datapusher.formats = csv xls xlsx
 
-.. todo:: Expand
+  ckan.datapusher.formats = csv xls
+
+Default value: ``csv xls application/csv application/vnd.ms-excel``
+
+File formats that will be pushed to the DataStore by the DataPusher. When
+adding or editing a resource which links to a file in one of these formats,
+the DataPusher will automatically try to import its contents to the DataStore.
+
 
 .. _ckan.datapusher.url:
 
 ckan.datapusher.url
 ^^^^^^^^^^^^^^^^^^^
 
 Example::
-  ckan.datapusher.url = http://datapusher.ckan.org/
 
-.. todo:: Expand
+  ckan.datapusher.url = http://127.0.0.1:8800/
+
+DataPusher endpoint to use when enabling the ``datapusher`` extension. If you
+installed CKAN via :doc:`install-from-package`, the DataPusher was installed for you
+running on port 8800. If you want to manually install the DataPusher, follow
+the installation `instructions <http://docs.ckan.org/projects/datapusher>`_.
 
 
 Activity Streams Settings

doc/datastore.rst

@@ -15,6 +15,11 @@ When a resource is added to the DataStore, you get:
 
 The DataStore is integrated into the :doc:`CKAN API <api>` and authorization system.
 
+The DataStore is generally used alongside the
+`DataPusher <http://docs.ckan.org/projects/datapusher>`_, which will
+automatically upload data to the DataStore from suitable files, whether
+uploaded to CKAN's FileStore or externally linked.
+
 .. contents::
    :depth: 1
    :local:
@@ -46,10 +51,6 @@ Setting up the DataStore
    available and the set-up is slightly different. Make sure, you read
    :ref:`legacy-mode` for more details.
 
-.. warning::
-
-   The DataStore does not support hiding resources in a private dataset.
-
 1. Enable the plugin
 ====================
 
@@ -126,7 +127,16 @@ Option 1: Paster command
 
 This option is preferred if CKAN and PostgreSQL are on the same server.
 
-To set the permissions, use this paster command after you've set the database URLs (make sure to have your virtualenv activated):
+To set the permissions, use the following paster command after you've set the database URLs.
+
+If you did a package install, the easiest way is to use the ``ckan`` command wrapper:
+
+.. parsed-literal::
+
+ sudo ckan datastore set-permissions postgres
+
+If you did a source install, make sure to have your virtualenv activated and
+run the command from the CKAN source directory:
 
 .. parsed-literal::
 
@@ -230,10 +240,13 @@ DataStore. This requires some processing, to extract the data from your files
 and to add it to the DataStore in the format the DataStore can handle.
 
 This task of automatically parsing and then adding data to the DataStore is
-performed by a DataPusher, a service that runs asynchronously and can be installed
-allongside CKAN.
+performed by the `DataPusher <DataPusher_docs>`_, a service that runs asynchronously and can be installed
+alongside CKAN.
+
+To install this please look at the docs here: http://docs.ckan.org/projects/datapusher
+
 
-To install this please look at the docs here: http://datapusher.readthedocs.org
+.. _DataPusher_docs: http://docs.ckan.org/projects/datapusher
 
 
 -----------------

doc/install-from-package.rst

@@ -9,6 +9,11 @@ and easiest way to install CKAN, but it requires **Ubuntu 12.04 64-bit**. If
 you're not using Ubuntu 12.04 64-bit, or if you're installing CKAN for
 development, you should follow :doc:`install-from-source` instead.
 
+At the end of the installation process you will end up with two running web
+applications, CKAN itself and the DataPusher, a separate service for automatically
+importing data to CKAN's :doc:`datastore`.
+
+
 .. _run-package-installer:
 
 ---------------------------
@@ -86,8 +91,8 @@ CKAN:
 
     sudo ckan db init
 
-#. Optionally, setup the DataStore by following the instructions in
-   :doc:`/datastore`.
+#. Optionally, setup the DataStore and DataPusher by following the
+   instructions in :doc:`/datastore`.
 
 #. Also optionally, you can enable file uploads by following the
    instructions in :doc:`filestore`.