From 1eb9d94077bd7d0dc586fefcb0d6b49042606b6c Mon Sep 17 00:00:00 2001
From: Sean Hammond <seanhammond@lavabit.com>
Date: Wed, 19 Jun 2013 15:27:33 +0200
Subject: [PATCH 01/87] [#1011] Fix a 500 to a 404

Catch Genshi TemplateNotFound errors when rendering the package read
page. This can happen if an IDatasetForm plugin returns a custom package
read template, e.g. 'read.html', but the user has requested the dataset
in RDF format and the plugin does not provide a corresponding 'read.rdf'
template. (CKAN will take the path to read.html, replace the filename
extension with rdf, try to render this non-existing template file, and
crash.)

Replace this crash with a proper 404.

Add to the IDatasetForm docs, explaining how to provide RDF templates
for datasets.

Fixes #1011.
---
 ckan/controllers/package.py | 11 ++++++++++-
 ckan/plugins/interfaces.py  |  9 +++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py
index 65541ee33bf..693ceb851a2 100644
--- a/ckan/controllers/package.py
+++ b/ckan/controllers/package.py
@@ -5,6 +5,7 @@
 from pylons import config
 from genshi.template import MarkupTemplate
 from genshi.template.text import NewTextTemplate
+import genshi.template.loader
 from paste.deploy.converters import asbool
 
 import ckan.logic as logic
@@ -352,7 +353,15 @@ def read(self, id, format='html'):
         template = self._read_template(package_type)
         template = template[:template.index('.') + 1] + format
 
-        return render(template, loader_class=loader)
+        try:
+            return render(template, loader_class=loader)
+        except genshi.template.loader.TemplateNotFound:
+            msg = _("Viewing {package_type} datasets in {format} format is "
+                    "not supported (template file {file} not found).".format(
+                    package_type=package_type, format=format, file=template))
+            abort(404, msg)
+
+        assert False, "We should never get here"
 
     def history(self, id):
         package_type = self._get_package_type(id.split('@')[0])
diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py
index a37d3c96ae8..826fec921cd 100644
--- a/ckan/plugins/interfaces.py
+++ b/ckan/plugins/interfaces.py
@@ -676,6 +676,15 @@ def read_template(self):
         The path should be relative to the plugin's templates dir, e.g.
         ``'package/read.html'``.
 
+        If the user requests the dataset in a format other than HTML
+        (CKAN supports returning datasets in RDF or N3 format by appending .rdf
+        or .n3 to the dataset read URL, see :doc:`linked-data-and-rdf`) then
+        CKAN will try to render
+        a template file with the same path as returned by this function,
+        but a different filename extension, e.g. ``'package/read.rdf'``.
+        If your extension doesn't have this RDF version of the template
+        file, the user will get a 404 error.
+
         :rtype: string
 
         '''

From 4d2d54a6e3622cdf4526ab59002e19114e7776c0 Mon Sep 17 00:00:00 2001
From: Sean Hammond <seanhammond@lavabit.com>
Date: Thu, 18 Jul 2013 13:03:55 +0200
Subject: [PATCH 02/87] [#1011] Refactor an exception raise and catch

Raise and catch a CKAN exception, instead of a Genshi one.
---
 ckan/controllers/package.py | 4 ++--
 ckan/lib/base.py            | 6 +-----
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py
index 693ceb851a2..d73ab4ee740 100644
--- a/ckan/controllers/package.py
+++ b/ckan/controllers/package.py
@@ -5,7 +5,6 @@
 from pylons import config
 from genshi.template import MarkupTemplate
 from genshi.template.text import NewTextTemplate
-import genshi.template.loader
 from paste.deploy.converters import asbool
 
 import ckan.logic as logic
@@ -20,6 +19,7 @@
 import ckan.lib.datapreview as datapreview
 import ckan.lib.plugins
 import ckan.plugins as p
+import lib.render
 
 from ckan.common import OrderedDict, _, json, request, c, g, response
 from home import CACHE_PARAMETERS
@@ -355,7 +355,7 @@ def read(self, id, format='html'):
 
         try:
             return render(template, loader_class=loader)
-        except genshi.template.loader.TemplateNotFound:
+        except lib.render.TemplateNotFound:
             msg = _("Viewing {package_type} datasets in {format} format is "
                     "not supported (template file {file} not found).".format(
                     package_type=package_type, format=format, file=template))
diff --git a/ckan/lib/base.py b/ckan/lib/base.py
index 7227734ddf4..914b42741ea 100644
--- a/ckan/lib/base.py
+++ b/ckan/lib/base.py
@@ -103,11 +103,7 @@ def render_template():
         # we remove it so any bad templates crash and burn
         del globs['url']
 
-        try:
-            template_path, template_type = lib.render.template_info(template_name)
-        except lib.render.TemplateNotFound:
-            template_type = 'genshi'
-            template_path = ''
+        template_path, template_type = lib.render.template_info(template_name)
 
         # snippets should not pass the context
         # but allow for legacy genshi templates

From 5b31702667badc49cc3b963506c3a0208ea24eda Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Tue, 13 Aug 2013 18:02:27 +0200
Subject: [PATCH 03/87] [#1183] Show very simple indicator for job status on
 resource page

---
 ckan/templates/package/resource_read.html |  1 +
 ckanext/datastore/logic/action.py         | 54 +++++++++++++++++++++++
 ckanext/datastore/logic/auth.py           |  4 ++
 ckanext/datastore/plugin.py               |  5 ++-
 4 files changed, 63 insertions(+), 1 deletion(-)

diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html
index 46406746a6f..025816c8923 100644
--- a/ckan/templates/package/resource_read.html
+++ b/ckan/templates/package/resource_read.html
@@ -26,6 +26,7 @@
           {% block resource_actions %}
           <ul>
             {% block resource_actions_inner %}
+            <li>{% snippet 'snippets/datapusher_status.html', resource=res %}</li>
             {% if h.check_access('package_update', {'id':pkg.id }) %}
               <li>{% link_for _('Edit'), controller='package', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn', icon='wrench' %}</li>
             {% endif %}
diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
index 4b79157355e..75ae3a163a8 100644
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -429,6 +429,13 @@ def datastore_make_public(context, data_dict):
     db.make_public(context, data_dict)
 
 
+def datapusher_enabled(context, data_dict):
+    """ Returns True if the DataPusher is configured """
+
+    datapusher_url = pylons.config.get('datapusher.url')
+    return True if datapusher_url else False
+
+
 def datapusher_submit(context, data_dict):
     ''' Submit a job to the datapusher. The datapusher is a service that
     imports tabular data into the datastore.
@@ -550,6 +557,53 @@ def datapusher_hook(context, data_dict):
     p.toolkit.get_action('task_status_update_many')(context, {'data': tasks})
 
 
+def datapusher_status(context, data_dict):
+    """ Get the status of a datapusher job for a certain resource.
+
+    :param resource_id: The resource id of the resource that you want the
+        datapusher status for.
+    :type resource_id: string
+    """
+
+    p.toolkit.check_access('datapusher_status', context, data_dict)
+
+    if 'id' in data_dict:
+        data_dict['resource_id'] = data_dict['id']
+    res_id = _get_or_bust(data_dict, 'resource_id')
+
+    try:
+        task_id = p.toolkit.get_action('task_status_show')(context, {
+            'entity_id': res_id,
+            'task_type': 'datapusher',
+            'key': 'job_id'
+        })
+    except p.toolkit.ObjectNotFound:
+        return {
+            'status': 'unknown'
+        }
+
+    task_key = p.toolkit.get_action('task_status_show')(context, {
+        'entity_id': res_id,
+        'task_type': 'datapusher',
+        'key': 'job_key'
+    })
+
+    datapusher_url = pylons.config.get('datapusher.url')
+    if not datapusher_url:
+        raise p.toolkit.ValidationError(
+            {'configuration': ['DataPusher not configured.']})
+
+    url = urlparse.urljoin(datapusher_url, 'job', task_id['value'])
+
+    return {
+        'status': task_id['state'],
+        'job_id': task_id['value'],
+        'job_url': url,
+        'last_updated': task_id['last_updated'],
+        'job_key': task_key['value']
+    }
+
+
 def _resource_exists(context, data_dict):
     # Returns true if the resource exists in CKAN and in the datastore
     model = _get_or_bust(context, 'model')
diff --git a/ckanext/datastore/logic/auth.py b/ckanext/datastore/logic/auth.py
index 4d044755035..c94a413382f 100644
--- a/ckanext/datastore/logic/auth.py
+++ b/ckanext/datastore/logic/auth.py
@@ -38,5 +38,9 @@ def datapusher_submit(context, data_dict):
     return _datastore_auth(context, data_dict)
 
 
+def datapusher_status(context, data_dict):
+    return _datastore_auth(context, data_dict)
+
+
 def datastore_change_permissions(context, data_dict):
     return _datastore_auth(context, data_dict)
diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py
index 4477ad70526..232c3e25829 100644
--- a/ckanext/datastore/plugin.py
+++ b/ckanext/datastore/plugin.py
@@ -256,8 +256,10 @@ def get_actions(self):
                    'datastore_upsert': action.datastore_upsert,
                    'datastore_delete': action.datastore_delete,
                    'datastore_search': action.datastore_search,
+                   'datapusher_enabled': action.datapusher_enabled,
                    'datapusher_submit': action.datapusher_submit,
                    'datapusher_hook': action.datapusher_hook,
+                   'datapusher_status': action.datapusher_status,
                    'resource_show': self.resource_show_action,
                   }
         if not self.legacy_mode:
@@ -273,7 +275,8 @@ def get_auth_functions(self):
                 'datastore_delete': auth.datastore_delete,
                 'datastore_search': auth.datastore_search,
                 'datastore_change_permissions': auth.datastore_change_permissions,
-                'datapusher_submit': auth.datapusher_submit}
+                'datapusher_submit': auth.datapusher_submit,
+                'datapusher_status': auth.datapusher_status}
 
     def before_map(self, m):
         m.connect('/datastore/dump/{resource_id}',

From d691059da2d71306a266e5ad5dd616a19e8b6b85 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Tue, 13 Aug 2013 18:24:02 +0200
Subject: [PATCH 04/87] [#1183] Fix url

---
 ckanext/datastore/logic/action.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
index 75ae3a163a8..2993bd9b083 100644
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -593,8 +593,7 @@ def datapusher_status(context, data_dict):
         raise p.toolkit.ValidationError(
             {'configuration': ['DataPusher not configured.']})
 
-    url = urlparse.urljoin(datapusher_url, 'job', task_id['value'])
-
+    url = urlparse.urljoin(datapusher_url, 'job' + '/' + task_id['value'])
     return {
         'status': task_id['state'],
         'job_id': task_id['value'],

From 64ba2315b3f987215b12880041b7aa4272185424 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Tue, 13 Aug 2013 18:38:11 +0200
Subject: [PATCH 05/87] [#938] Fix datapusher test

---
 ckanext/datastore/tests/test_create.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py
index 589a6c26e3a..d7b893235ed 100644
--- a/ckanext/datastore/tests/test_create.py
+++ b/ckanext/datastore/tests/test_create.py
@@ -556,6 +556,7 @@ def test_create_ckan_resource_in_package(self):
 
     @httpretty.activate
     def test_providing_res_with_url_calls_datapusher_correctly(self):
+        pylons.config['datapusher.url'] = 'http://datapusher.ckan.org'
         httpretty.HTTPretty.register_uri(
             httpretty.HTTPretty.POST,
             'http://datapusher.ckan.org/job',

From f367fd2ae3f00300895e352f2733b1e91489fc79 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Tue, 13 Aug 2013 23:36:44 +0200
Subject: [PATCH 06/87] [#1183] Add snippet for datapusher status indicator

---
 ckan/public/base/less/ckan.less               |  1 +
 ckan/public/base/less/datapusher.less         | 18 ++++++++++++++++++
 .../templates/snippets/datapusher_status.html | 19 +++++++++++++++++++
 3 files changed, 38 insertions(+)
 create mode 100644 ckan/public/base/less/datapusher.less
 create mode 100644 ckan/templates/snippets/datapusher_status.html

diff --git a/ckan/public/base/less/ckan.less b/ckan/public/base/less/ckan.less
index f7c108f8734..f842bd6dbae 100644
--- a/ckan/public/base/less/ckan.less
+++ b/ckan/public/base/less/ckan.less
@@ -20,6 +20,7 @@
 @import "activity.less";
 @import "dropdown.less";
 @import "dashboard.less";
+@import "datapusher.less";
 
 body {
   // Using the masthead/footer gradient prevents the color from changing
diff --git a/ckan/public/base/less/datapusher.less b/ckan/public/base/less/datapusher.less
new file mode 100644
index 00000000000..f147c0c3f6c
--- /dev/null
+++ b/ckan/public/base/less/datapusher.less
@@ -0,0 +1,18 @@
+.datapusher-status-link:hover {
+	text-decoration: none;
+}
+
+.datapusher-status {
+	&.status-unknown {
+		color: #bbb;
+	}
+	&.status-pending {
+		color: #FFCC00;
+	}
+	&.status-error {
+		color: red;
+	}
+	&.status-complete {
+		color: #009900;
+	}
+}
\ No newline at end of file
diff --git a/ckan/templates/snippets/datapusher_status.html b/ckan/templates/snippets/datapusher_status.html
new file mode 100644
index 00000000000..bb95b08d809
--- /dev/null
+++ b/ckan/templates/snippets/datapusher_status.html
@@ -0,0 +1,19 @@
+{# Datapusher status indicator
+
+resource: the resource
+
+#}
+{% if resource.datastore_active %}
+  {% set datapusher_enabled = h.get_action('datapusher_enabled') %}
+	{% if datapusher_enabled %}
+		{% set job = h.get_action('datapusher_status', data_dict={'resource_id': resource.id}) %}
+	  {% set status = job.status %}
+	  {% set datapusher_job_url = job.job_url %}
+	  {% set title =  _('Datapusher status: {status}').format(status=status) %}
+	  {% if status == 'unknown' %}
+	  	<i title="{{ _(title) }}" class="datapusher-status status-{{ status }} icon-circle icon-large"></i>
+	  {% else %}
+	  	<a href="{{ datapusher_job_url }}" class="datapusher-status-link" title="{{ _(title) }}"><i class="datapusher-status status-{{ status }} icon-circle icon-large"></i></a>
+  	{% endif %}
+  {% endif %}
+{% endif %}

From afb105999e4b6a49ea1f54bcb122c4278c9648ff Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Tue, 13 Aug 2013 23:45:09 +0200
Subject: [PATCH 07/87] [#1183] Clean up template code

---
 ckan/templates/snippets/datapusher_status.html | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/ckan/templates/snippets/datapusher_status.html b/ckan/templates/snippets/datapusher_status.html
index bb95b08d809..d5ac0108520 100644
--- a/ckan/templates/snippets/datapusher_status.html
+++ b/ckan/templates/snippets/datapusher_status.html
@@ -7,13 +7,11 @@
   {% set datapusher_enabled = h.get_action('datapusher_enabled') %}
 	{% if datapusher_enabled %}
 		{% set job = h.get_action('datapusher_status', data_dict={'resource_id': resource.id}) %}
-	  {% set status = job.status %}
-	  {% set datapusher_job_url = job.job_url %}
-	  {% set title =  _('Datapusher status: {status}').format(status=status) %}
-	  {% if status == 'unknown' %}
-	  	<i title="{{ _(title) }}" class="datapusher-status status-{{ status }} icon-circle icon-large"></i>
+	  {% set title =  _('Datapusher status: {status}.').format(status=job.status) %}
+	  {% if job.status == 'unknown' %}
+	  	<i title="{{ _(title) }}" class="datapusher-status status-{{ job.status }} icon-circle icon-large"></i>
 	  {% else %}
-	  	<a href="{{ datapusher_job_url }}" class="datapusher-status-link" title="{{ _(title) }}"><i class="datapusher-status status-{{ status }} icon-circle icon-large"></i></a>
+	  	<a href="{{ job.job_url }}" class="datapusher-status-link" title="{{ _(title) }}"><i class="datapusher-status status-{{ job.status }} icon-circle icon-large"></i></a>
   	{% endif %}
   {% endif %}
 {% endif %}

From 77060c2ac1d24f13ce67d9ac9ea7294e92985f67 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Tue, 13 Aug 2013 23:47:43 +0200
Subject: [PATCH 08/87] [#1183] Only show for users who can edit resource

---
 ckan/templates/package/resource_read.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html
index 025816c8923..d4433d4a4a8 100644
--- a/ckan/templates/package/resource_read.html
+++ b/ckan/templates/package/resource_read.html
@@ -26,8 +26,8 @@
           {% block resource_actions %}
           <ul>
             {% block resource_actions_inner %}
-            <li>{% snippet 'snippets/datapusher_status.html', resource=res %}</li>
             {% if h.check_access('package_update', {'id':pkg.id }) %}
+              <li>{% snippet 'snippets/datapusher_status.html', resource=res %}</li>
               <li>{% link_for _('Edit'), controller='package', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn', icon='wrench' %}</li>
             {% endif %}
             {% if res.url %}

From 78a289e119591e3d8c0972f0fab8447855b73f10 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Thu, 15 Aug 2013 20:32:34 +0200
Subject: [PATCH 09/87] [#1183] Don't use get_action because it will be
 deprecated in #1193

---
 ckan/templates/snippets/datapusher_status.html |  5 ++---
 ckanext/datastore/logic/action.py              | 15 +++++----------
 ckanext/datastore/plugin.py                    |  7 +++++++
 3 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/ckan/templates/snippets/datapusher_status.html b/ckan/templates/snippets/datapusher_status.html
index d5ac0108520..863faeee491 100644
--- a/ckan/templates/snippets/datapusher_status.html
+++ b/ckan/templates/snippets/datapusher_status.html
@@ -4,9 +4,8 @@
 
 #}
 {% if resource.datastore_active %}
-  {% set datapusher_enabled = h.get_action('datapusher_enabled') %}
-	{% if datapusher_enabled %}
-		{% set job = h.get_action('datapusher_status', data_dict={'resource_id': resource.id}) %}
+	{% if h.datapusher_enabled() %}
+		{% set job = h.datapusher_status(resource.id) %}
 	  {% set title =  _('Datapusher status: {status}.').format(status=job.status) %}
 	  {% if job.status == 'unknown' %}
 	  	<i title="{{ _(title) }}" class="datapusher-status status-{{ job.status }} icon-circle icon-large"></i>
diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
index 2993bd9b083..8aa76493a6a 100644
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -571,16 +571,11 @@ def datapusher_status(context, data_dict):
         data_dict['resource_id'] = data_dict['id']
     res_id = _get_or_bust(data_dict, 'resource_id')
 
-    try:
-        task_id = p.toolkit.get_action('task_status_show')(context, {
-            'entity_id': res_id,
-            'task_type': 'datapusher',
-            'key': 'job_id'
-        })
-    except p.toolkit.ObjectNotFound:
-        return {
-            'status': 'unknown'
-        }
+    task_id = p.toolkit.get_action('task_status_show')(context, {
+        'entity_id': res_id,
+        'task_type': 'datapusher',
+        'key': 'job_id'
+    })
 
     task_key = p.toolkit.get_action('task_status_show')(context, {
         'entity_id': res_id,
diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py
index 232c3e25829..b24dc30446b 100644
--- a/ckanext/datastore/plugin.py
+++ b/ckanext/datastore/plugin.py
@@ -5,6 +5,7 @@
 import ckanext.datastore.logic.action as action
 import ckanext.datastore.logic.auth as auth
 import ckanext.datastore.db as db
+import ckanext.datastore.helpers as helpers
 import ckan.logic as logic
 import ckan.model as model
 
@@ -26,6 +27,7 @@ class DatastorePlugin(p.SingletonPlugin):
     p.implements(p.IDomainObjectModification, inherit=True)
     p.implements(p.IRoutes, inherit=True)
     p.implements(p.IResourceController, inherit=True)
+    p.implements(p.ITemplateHelpers)
 
     legacy_mode = False
     resource_show_action = None
@@ -293,3 +295,8 @@ def before_show(self, resource_dict):
                 controller='ckanext.datastore.controller:DatastoreController',
                 action='dump', resource_id=resource_dict['id'])
         return resource_dict
+
+    def get_helpers(self):
+        return {
+            'datapusher_status': helpers.datapusher_status,
+            'datapusher_enabled': helpers.datapusher_enabled}

From 8c85e3aeee07fa808cae3e0f532809acd804a780 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Thu, 15 Aug 2013 20:45:37 +0200
Subject: [PATCH 10/87] [#938] Use datapusher_enabled action instead of
 checking whether it is enabled ourselves

---
 ckanext/datastore/logic/action.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
index 8aa76493a6a..b57dd982b2b 100644
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -460,12 +460,11 @@ def datapusher_submit(context, data_dict):
 
     p.toolkit.check_access('datapusher_submit', context, data_dict)
 
-    datapusher_url = pylons.config.get('datapusher.url')
-
-    # no datapusher url means the datapusher should not be used
-    if not datapusher_url:
+    if not p.toolkit.get_action('datapusher_enabled')(context):
         return False
 
+    datapusher_url = pylons.config.get('datapusher.url')
+
     callback_url = p.toolkit.url_for(
         controller='api', action='action', logic_function='datapusher_hook',
         ver=3, qualified=True)

From deeb6efb7fec700516a97d5030a1ab0a69b2a8d1 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Thu, 15 Aug 2013 20:48:20 +0200
Subject: [PATCH 11/87] [#1183] Add file that I forgot

---
 ckanext/datastore/helpers.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 ckanext/datastore/helpers.py

diff --git a/ckanext/datastore/helpers.py b/ckanext/datastore/helpers.py
new file mode 100644
index 00000000000..bf06b2e063a
--- /dev/null
+++ b/ckanext/datastore/helpers.py
@@ -0,0 +1,15 @@
+import ckan.plugins.toolkit as toolkit
+
+
+def datapusher_status(resource_id):
+    try:
+        return toolkit.get_action('datapusher_status')(
+            {}, {'resource_id': resource_id})
+    except toolkit.ObjectNotFound:
+        return {
+            'status': 'unknown'
+        }
+
+
+def datapusher_enabled():
+    return toolkit.get_action('datapusher_enabled')({})

From 236a8e927c78597d3e73e16dd0a2eba90d458d81 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Thu, 15 Aug 2013 20:48:20 +0200
Subject: [PATCH 12/87] [#1183] Add file that I forgot

---
 ckanext/datastore/helpers.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 ckanext/datastore/helpers.py

diff --git a/ckanext/datastore/helpers.py b/ckanext/datastore/helpers.py
new file mode 100644
index 00000000000..bf06b2e063a
--- /dev/null
+++ b/ckanext/datastore/helpers.py
@@ -0,0 +1,15 @@
+import ckan.plugins.toolkit as toolkit
+
+
+def datapusher_status(resource_id):
+    try:
+        return toolkit.get_action('datapusher_status')(
+            {}, {'resource_id': resource_id})
+    except toolkit.ObjectNotFound:
+        return {
+            'status': 'unknown'
+        }
+
+
+def datapusher_enabled():
+    return toolkit.get_action('datapusher_enabled')({})

From 7d0b22b560a4d3b96d7e64d41e375f501acef73b Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Thu, 15 Aug 2013 21:17:45 +0200
Subject: [PATCH 13/87] [#1196] Move datapusher code into separate extension

---
 ckan/config/deployment.ini_tmpl              |   1 +
 ckanext/datapusher/__init__.py               |   0
 ckanext/{datastore => datapusher}/helpers.py |   0
 ckanext/datapusher/logic/__init__.py         |   0
 ckanext/datapusher/logic/action.py           | 183 +++++++++++++++++++
 ckanext/datapusher/logic/auth.py             |   9 +
 ckanext/datapusher/plugin.py                 |  78 ++++++++
 ckanext/datastore/logic/action.py            | 168 -----------------
 ckanext/datastore/logic/auth.py              |  20 +-
 ckanext/datastore/plugin.py                  |  57 +-----
 ckanext/datastore/tests/test_create.py       |   2 +
 setup.py                                     |   1 +
 12 files changed, 284 insertions(+), 235 deletions(-)
 create mode 100644 ckanext/datapusher/__init__.py
 rename ckanext/{datastore => datapusher}/helpers.py (100%)
 create mode 100644 ckanext/datapusher/logic/__init__.py
 create mode 100644 ckanext/datapusher/logic/action.py
 create mode 100644 ckanext/datapusher/logic/auth.py
 create mode 100644 ckanext/datapusher/plugin.py

diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl
index 97c4f9412ba..0729c606f3c 100644
--- a/ckan/config/deployment.ini_tmpl
+++ b/ckan/config/deployment.ini_tmpl
@@ -79,6 +79,7 @@ ckan.site_id = default
 ## Plugins Settings
 
 # Note: Add ``datastore`` to enable the CKAN DataStore
+#       Add ``datapusher`` to enable DataPusher
 #       Add ``pdf_preview`` to enable the resource preview for PDFs
 #		Add ``resource_proxy`` to enable resorce proxying and get around the
 #		same origin policy
diff --git a/ckanext/datapusher/__init__.py b/ckanext/datapusher/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/ckanext/datastore/helpers.py b/ckanext/datapusher/helpers.py
similarity index 100%
rename from ckanext/datastore/helpers.py
rename to ckanext/datapusher/helpers.py
diff --git a/ckanext/datapusher/logic/__init__.py b/ckanext/datapusher/logic/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py
new file mode 100644
index 00000000000..5957887d44e
--- /dev/null
+++ b/ckanext/datapusher/logic/action.py
@@ -0,0 +1,183 @@
+import logging
+import json
+import urlparse
+import datetime
+
+import pylons
+import requests
+
+import ckan.lib.navl.dictization_functions
+import ckan.logic as logic
+import ckan.plugins as p
+
+log = logging.getLogger(__name__)
+_get_or_bust = logic.get_or_bust
+_validate = ckan.lib.navl.dictization_functions.validate
+
+
+def datapusher_enabled(context, data_dict):
+    """ Returns True if the DataPusher is configured """
+
+    datapusher_url = pylons.config.get('datapusher.url')
+    return True if datapusher_url else False
+
+
+def datapusher_submit(context, data_dict):
+    ''' Submit a job to the datapusher. The datapusher is a service that
+    imports tabular data into the datastore.
+
+    :param resource_id: The resource id of the resource that the data
+        should be imported in. The resource's URL will be used to get the data.
+    :type resource_id: string
+    :param set_url_type: If set to true, the ``url_type`` of the resource will
+        be set to ``datastore`` and the resource URL will automatically point
+        to the :ref:`datastore dump <dump>` URL. (optional, default: False)
+    :type set_url_type: boolean
+
+    Returns ``True`` if the job has been submitted and ``False`` if the job
+    has not been submitted, i.e. when the datapusher is not configured.
+
+    :rtype: boolean
+    '''
+
+    if 'id' in data_dict:
+        data_dict['resource_id'] = data_dict['id']
+    res_id = _get_or_bust(data_dict, 'resource_id')
+
+    p.toolkit.check_access('datapusher_submit', context, data_dict)
+
+    if not p.toolkit.get_action('datapusher_enabled')(context):
+        return False
+
+    datapusher_url = pylons.config.get('datapusher.url')
+
+    callback_url = p.toolkit.url_for(
+        controller='api', action='action', logic_function='datapusher_hook',
+        ver=3, qualified=True)
+
+    user = p.toolkit.get_action('user_show')(context, {'id': context['user']})
+    try:
+        r = requests.post(
+            urlparse.urljoin(datapusher_url, 'job'),
+            headers={
+                'Content-Type': 'application/json'
+            },
+            data=json.dumps({
+                'api_key': user['apikey'],
+                'job_type': 'push_to_datastore',
+                'result_url': callback_url,
+                'metadata': {
+                    'ckan_url': pylons.config['ckan.site_url'],
+                    'resource_id': res_id,
+                    'set_url_type': data_dict.get('set_url_type', False)
+                }
+            }))
+        r.raise_for_status()
+    except requests.exceptions.ConnectionError, e:
+        raise p.toolkit.ValidationError({'datapusher': {
+            'message': 'Could not connect to DataPusher.',
+            'details': str(e)}})
+    except requests.exceptions.HTTPError, e:
+        m = 'An Error occurred while sending the job: {0}'.format(e.message)
+        try:
+            body = e.response.json()
+        except ValueError:
+            body = e.response.text
+        raise p.toolkit.ValidationError({'datapusher': {
+            'message': m,
+            'details': body,
+            'status_code': r.status_code}})
+
+    empty_task = {
+        'entity_id': res_id,
+        'entity_type': 'resource',
+        'task_type': 'datapusher',
+        'last_updated': str(datetime.datetime.now()),
+        'state': 'pending'
+    }
+
+    tasks = []
+    for (k, v) in [('job_id', r.json()['job_id']),
+                   ('job_key', r.json()['job_key'])]:
+        t = empty_task.copy()
+        t['key'] = k
+        t['value'] = v
+        tasks.append(t)
+    p.toolkit.get_action('task_status_update_many')(context, {'data': tasks})
+
+    return True
+
+
+def datapusher_hook(context, data_dict):
+    """ Update datapusher task. This action is typically called by the
+    datapusher whenever the status of a job changes.
+
+    Expects a job with ``status`` and ``metadata`` with a ``resource_id``.
+    """
+
+    # TODO: use a schema to validate
+
+    p.toolkit.check_access('datapusher_submit', context, data_dict)
+
+    res_id = data_dict['metadata']['resource_id']
+
+    task_id = p.toolkit.get_action('task_status_show')(context, {
+        'entity_id': res_id,
+        'task_type': 'datapusher',
+        'key': 'job_id'
+    })
+
+    task_key = p.toolkit.get_action('task_status_show')(context, {
+        'entity_id': res_id,
+        'task_type': 'datapusher',
+        'key': 'job_key'
+    })
+
+    tasks = [task_id, task_key]
+
+    for task in tasks:
+        task['state'] = data_dict['status']
+        task['last_updated'] = str(datetime.datetime.now())
+
+    p.toolkit.get_action('task_status_update_many')(context, {'data': tasks})
+
+
+def datapusher_status(context, data_dict):
+    """ Get the status of a datapusher job for a certain resource.
+
+    :param resource_id: The resource id of the resource that you want the
+        datapusher status for.
+    :type resource_id: string
+    """
+
+    p.toolkit.check_access('datapusher_status', context, data_dict)
+
+    if 'id' in data_dict:
+        data_dict['resource_id'] = data_dict['id']
+    res_id = _get_or_bust(data_dict, 'resource_id')
+
+    task_id = p.toolkit.get_action('task_status_show')(context, {
+        'entity_id': res_id,
+        'task_type': 'datapusher',
+        'key': 'job_id'
+    })
+
+    task_key = p.toolkit.get_action('task_status_show')(context, {
+        'entity_id': res_id,
+        'task_type': 'datapusher',
+        'key': 'job_key'
+    })
+
+    datapusher_url = pylons.config.get('datapusher.url')
+    if not datapusher_url:
+        raise p.toolkit.ValidationError(
+            {'configuration': ['DataPusher not configured.']})
+
+    url = urlparse.urljoin(datapusher_url, 'job' + '/' + task_id['value'])
+    return {
+        'status': task_id['state'],
+        'job_id': task_id['value'],
+        'job_url': url,
+        'last_updated': task_id['last_updated'],
+        'job_key': task_key['value']
+    }
diff --git a/ckanext/datapusher/logic/auth.py b/ckanext/datapusher/logic/auth.py
new file mode 100644
index 00000000000..55a7e832488
--- /dev/null
+++ b/ckanext/datapusher/logic/auth.py
@@ -0,0 +1,9 @@
+import ckanext.datastore.logic.auth as auth
+
+
+def datapusher_submit(context, data_dict):
+    return auth.datastore_auth(context, data_dict)
+
+
+def datapusher_status(context, data_dict):
+    return auth.datastore_auth(context, data_dict)
diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py
new file mode 100644
index 00000000000..17f9022fd22
--- /dev/null
+++ b/ckanext/datapusher/plugin.py
@@ -0,0 +1,78 @@
+import logging
+
+import ckan.plugins as p
+import ckanext.datapusher.logic.action as action
+import ckanext.datapusher.logic.auth as auth
+import ckanext.datapusher.helpers as helpers
+import ckan.logic as logic
+import ckan.model as model
+
+log = logging.getLogger(__name__)
+_get_or_bust = logic.get_or_bust
+
+DEFAULT_FORMATS = []
+
+
+class DatastoreException(Exception):
+    pass
+
+
+class DatapusherPlugin(p.SingletonPlugin):
+    p.implements(p.IConfigurable, inherit=True)
+    p.implements(p.IActions)
+    p.implements(p.IAuthFunctions)
+    p.implements(p.IResourceUrlChange)
+    p.implements(p.IDomainObjectModification, inherit=True)
+    p.implements(p.IResourceController, inherit=True)
+    p.implements(p.ITemplateHelpers)
+
+    legacy_mode = False
+    resource_show_action = None
+
+    def configure(self, config):
+        self.config = config
+
+        datapusher_formats = config.get('datapusher.formats', '').split()
+        self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS
+
+    def notify(self, entity, operation=None):
+        if isinstance(entity, model.Resource):
+            if (operation == model.domain_object.DomainObjectOperation.new
+                    or not operation):
+                # if operation is None, resource URL has been changed, as
+                # the notify function in IResourceUrlChange only takes
+                # 1 parameter
+                context = {'model': model, 'ignore_auth': True}
+                package = p.toolkit.get_action('package_show')(context, {
+                    'id': entity.get_package_id()
+                })
+                if (not package['private'] and
+                        entity.format in self.datapusher_formats):
+                    p.toolkit.get_action('datapusher_submit')(context, {
+                        'resource_id': entity.id
+                    })
+
+    def get_actions(self):
+        return {'datapusher_enabled': action.datapusher_enabled,
+                'datapusher_submit': action.datapusher_submit,
+                'datapusher_hook': action.datapusher_hook,
+                'datapusher_status': action.datapusher_status}
+
+    def get_auth_functions(self):
+        return {'datapusher_submit': auth.datapusher_submit,
+                'datapusher_status': auth.datapusher_status}
+
+    def before_show(self, resource_dict):
+        ''' Modify the resource url of datastore resources so that
+        they link to the datastore dumps.
+        '''
+        if resource_dict['url_type'] == 'datastore':
+            resource_dict['url'] = p.toolkit.url_for(
+                controller='ckanext.datastore.controller:DatastoreController',
+                action='dump', resource_id=resource_dict['id'])
+        return resource_dict
+
+    def get_helpers(self):
+        return {
+            'datapusher_status': helpers.datapusher_status,
+            'datapusher_enabled': helpers.datapusher_enabled}
diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
index b57dd982b2b..9e93deed315 100644
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -429,174 +429,6 @@ def datastore_make_public(context, data_dict):
     db.make_public(context, data_dict)
 
 
-def datapusher_enabled(context, data_dict):
-    """ Returns True if the DataPusher is configured """
-
-    datapusher_url = pylons.config.get('datapusher.url')
-    return True if datapusher_url else False
-
-
-def datapusher_submit(context, data_dict):
-    ''' Submit a job to the datapusher. The datapusher is a service that
-    imports tabular data into the datastore.
-
-    :param resource_id: The resource id of the resource that the data
-        should be imported in. The resource's URL will be used to get the data.
-    :type resource_id: string
-    :param set_url_type: If set to true, the ``url_type`` of the resource will
-        be set to ``datastore`` and the resource URL will automatically point
-        to the :ref:`datastore dump <dump>` URL. (optional, default: False)
-    :type set_url_type: boolean
-
-    Returns ``True`` if the job has been submitted and ``False`` if the job
-    has not been submitted, i.e. when the datapusher is not configured.
-
-    :rtype: boolean
-    '''
-
-    if 'id' in data_dict:
-        data_dict['resource_id'] = data_dict['id']
-    res_id = _get_or_bust(data_dict, 'resource_id')
-
-    p.toolkit.check_access('datapusher_submit', context, data_dict)
-
-    if not p.toolkit.get_action('datapusher_enabled')(context):
-        return False
-
-    datapusher_url = pylons.config.get('datapusher.url')
-
-    callback_url = p.toolkit.url_for(
-        controller='api', action='action', logic_function='datapusher_hook',
-        ver=3, qualified=True)
-
-    user = p.toolkit.get_action('user_show')(context, {'id': context['user']})
-    try:
-        r = requests.post(
-            urlparse.urljoin(datapusher_url, 'job'),
-            headers={
-                'Content-Type': 'application/json'
-            },
-            data=json.dumps({
-                'api_key': user['apikey'],
-                'job_type': 'push_to_datastore',
-                'result_url': callback_url,
-                'metadata': {
-                    'ckan_url': pylons.config['ckan.site_url'],
-                    'resource_id': res_id,
-                    'set_url_type': data_dict.get('set_url_type', False)
-                }
-            }))
-        r.raise_for_status()
-    except requests.exceptions.ConnectionError, e:
-        raise p.toolkit.ValidationError({'datapusher': {
-            'message': 'Could not connect to DataPusher.',
-            'details': str(e)}})
-    except requests.exceptions.HTTPError, e:
-        m = 'An Error occurred while sending the job: {0}'.format(e.message)
-        try:
-            body = e.response.json()
-        except ValueError:
-            body = e.response.text
-        raise p.toolkit.ValidationError({'datapusher': {
-            'message': m,
-            'details': body,
-            'status_code': r.status_code}})
-
-    empty_task = {
-        'entity_id': res_id,
-        'entity_type': 'resource',
-        'task_type': 'datapusher',
-        'last_updated': str(datetime.datetime.now()),
-        'state': 'pending'
-    }
-
-    tasks = []
-    for (k, v) in [('job_id', r.json()['job_id']),
-                   ('job_key', r.json()['job_key'])]:
-        t = empty_task.copy()
-        t['key'] = k
-        t['value'] = v
-        tasks.append(t)
-    p.toolkit.get_action('task_status_update_many')(context, {'data': tasks})
-
-    return True
-
-
-def datapusher_hook(context, data_dict):
-    """ Update datapusher task. This action is typically called by the
-    datapusher whenever the status of a job changes.
-
-    Expects a job with ``status`` and ``metadata`` with a ``resource_id``.
-    """
-
-    # TODO: use a schema to validate
-
-    p.toolkit.check_access('datapusher_submit', context, data_dict)
-
-    res_id = data_dict['metadata']['resource_id']
-
-    task_id = p.toolkit.get_action('task_status_show')(context, {
-        'entity_id': res_id,
-        'task_type': 'datapusher',
-        'key': 'job_id'
-    })
-
-    task_key = p.toolkit.get_action('task_status_show')(context, {
-        'entity_id': res_id,
-        'task_type': 'datapusher',
-        'key': 'job_key'
-    })
-
-    tasks = [task_id, task_key]
-
-    for task in tasks:
-        task['state'] = data_dict['status']
-        task['last_updated'] = str(datetime.datetime.now())
-
-    p.toolkit.get_action('task_status_update_many')(context, {'data': tasks})
-
-
-def datapusher_status(context, data_dict):
-    """ Get the status of a datapusher job for a certain resource.
-
-    :param resource_id: The resource id of the resource that you want the
-        datapusher status for.
-    :type resource_id: string
-    """
-
-    p.toolkit.check_access('datapusher_status', context, data_dict)
-
-    if 'id' in data_dict:
-        data_dict['resource_id'] = data_dict['id']
-    res_id = _get_or_bust(data_dict, 'resource_id')
-
-    task_id = p.toolkit.get_action('task_status_show')(context, {
-        'entity_id': res_id,
-        'task_type': 'datapusher',
-        'key': 'job_id'
-    })
-
-    task_key = p.toolkit.get_action('task_status_show')(context, {
-        'entity_id': res_id,
-        'task_type': 'datapusher',
-        'key': 'job_key'
-    })
-
-    datapusher_url = pylons.config.get('datapusher.url')
-    if not datapusher_url:
-        raise p.toolkit.ValidationError(
-            {'configuration': ['DataPusher not configured.']})
-
-    url = urlparse.urljoin(datapusher_url, 'job' + '/' + task_id['value'])
-    return {
-        'status': task_id['state'],
-        'job_id': task_id['value'],
-        'job_url': url,
-        'last_updated': task_id['last_updated'],
-        'job_key': task_key['value']
-    }
-
-
 def _resource_exists(context, data_dict):
     # Returns true if the resource exists in CKAN and in the datastore
     model = _get_or_bust(context, 'model')
diff --git a/ckanext/datastore/logic/auth.py b/ckanext/datastore/logic/auth.py
index c94a413382f..357ae97fd6b 100644
--- a/ckanext/datastore/logic/auth.py
+++ b/ckanext/datastore/logic/auth.py
@@ -1,7 +1,7 @@
 import ckan.plugins as p
 
 
-def _datastore_auth(context, data_dict, privilege='resource_update'):
+def datastore_auth(context, data_dict, privilege='resource_update'):
     if not 'id' in data_dict:
         data_dict['id'] = data_dict.get('resource_id')
     user = context.get('user')
@@ -19,28 +19,20 @@ def _datastore_auth(context, data_dict, privilege='resource_update'):
 
 
 def datastore_create(context, data_dict):
-    return _datastore_auth(context, data_dict)
+    return datastore_auth(context, data_dict)
 
 
 def datastore_upsert(context, data_dict):
-    return _datastore_auth(context, data_dict)
+    return datastore_auth(context, data_dict)
 
 
 def datastore_delete(context, data_dict):
-    return _datastore_auth(context, data_dict)
+    return datastore_auth(context, data_dict)
 
 
 def datastore_search(context, data_dict):
-    return _datastore_auth(context, data_dict, 'resource_show')
-
-
-def datapusher_submit(context, data_dict):
-    return _datastore_auth(context, data_dict)
-
-
-def datapusher_status(context, data_dict):
-    return _datastore_auth(context, data_dict)
+    return datastore_auth(context, data_dict, 'resource_show')
 
 
 def datastore_change_permissions(context, data_dict):
-    return _datastore_auth(context, data_dict)
+    return datastore_auth(context, data_dict)
diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py
index b24dc30446b..8eb753f8e46 100644
--- a/ckanext/datastore/plugin.py
+++ b/ckanext/datastore/plugin.py
@@ -5,7 +5,6 @@
 import ckanext.datastore.logic.action as action
 import ckanext.datastore.logic.auth as auth
 import ckanext.datastore.db as db
-import ckanext.datastore.helpers as helpers
 import ckan.logic as logic
 import ckan.model as model
 
@@ -26,8 +25,6 @@ class DatastorePlugin(p.SingletonPlugin):
     p.implements(p.IResourceUrlChange)
     p.implements(p.IDomainObjectModification, inherit=True)
     p.implements(p.IRoutes, inherit=True)
-    p.implements(p.IResourceController, inherit=True)
-    p.implements(p.ITemplateHelpers)
 
     legacy_mode = False
     resource_show_action = None
@@ -104,36 +101,11 @@ def new_resource_show(context, data_dict):
             self.resource_show_action = new_resource_show
 
     def notify(self, entity, operation=None):
-        '''
-        if not isinstance(entity, model.Resource):
-            return
-        if operation:
-            if operation == model.domain_object.DomainObjectOperation.new:
-                self._create_datastorer_task(entity)
-        else:
-            # if operation is None, resource URL has been changed, as the
-            # notify function in IResourceUrlChange only takes 1 parameter
-            self._create_datastorer_task(entity)
-        '''
-        context = {'model': model, 'ignore_auth': True}
-        if isinstance(entity, model.Resource):
-            if (operation == model.domain_object.DomainObjectOperation.new
-                    or not operation):
-                # if operation is None, resource URL has been changed, as
-                # the notify function in IResourceUrlChange only takes
-                # 1 parameter
-                package = p.toolkit.get_action('package_show')(context, {
-                    'id': entity.get_package_id()
-                })
-                if (not package['private'] and
-                        entity.format in self.datapusher_formats):
-                    p.toolkit.get_action('datapusher_submit')(context, {
-                        'resource_id': entity.id
-                    })
         if not isinstance(entity, model.Package) or self.legacy_mode:
             return
         # if a resource is new, it cannot have a datastore resource, yet
         if operation == model.domain_object.DomainObjectOperation.changed:
+            context = {'model': model, 'ignore_auth': True}
             if entity.private:
                 func = p.toolkit.get_action('datastore_make_private')
             else:
@@ -160,7 +132,7 @@ def _check_urls_and_permissions(self):
             self._log_or_raise('CKAN and DataStore database '
                                'cannot be the same.')
 
-        # in legacy mode, the read and write url are ths same (both write url)
+        # in legacy mode, the read and write url are the same (both write url)
         # consequently the same url check and and write privilege check
         # don't make sense
         if not self.legacy_mode:
@@ -258,10 +230,6 @@ def get_actions(self):
                    'datastore_upsert': action.datastore_upsert,
                    'datastore_delete': action.datastore_delete,
                    'datastore_search': action.datastore_search,
-                   'datapusher_enabled': action.datapusher_enabled,
-                   'datapusher_submit': action.datapusher_submit,
-                   'datapusher_hook': action.datapusher_hook,
-                   'datapusher_status': action.datapusher_status,
                    'resource_show': self.resource_show_action,
                   }
         if not self.legacy_mode:
@@ -276,27 +244,10 @@ def get_auth_functions(self):
                 'datastore_upsert': auth.datastore_upsert,
                 'datastore_delete': auth.datastore_delete,
                 'datastore_search': auth.datastore_search,
-                'datastore_change_permissions': auth.datastore_change_permissions,
-                'datapusher_submit': auth.datapusher_submit,
-                'datapusher_status': auth.datapusher_status}
+                'datastore_change_permissions': auth.datastore_change_permissions}
 
     def before_map(self, m):
         m.connect('/datastore/dump/{resource_id}',
                   controller='ckanext.datastore.controller:DatastoreController',
                   action='dump')
-        return m
-
-    def before_show(self, resource_dict):
-        ''' Modify the resource url of datastore resources so that
-        they link to the datastore dumps.
-        '''
-        if resource_dict['url_type'] == 'datastore':
-            resource_dict['url'] = p.toolkit.url_for(
-                controller='ckanext.datastore.controller:DatastoreController',
-                action='dump', resource_id=resource_dict['id'])
-        return resource_dict
-
-    def get_helpers(self):
-        return {
-            'datapusher_status': helpers.datapusher_status,
-            'datapusher_enabled': helpers.datapusher_enabled}
+        return m
\ No newline at end of file
diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py
index d7b893235ed..7a6933008e1 100644
--- a/ckanext/datastore/tests/test_create.py
+++ b/ckanext/datastore/tests/test_create.py
@@ -38,6 +38,7 @@ def setup_class(cls):
         if not tests.is_datastore_supported():
             raise nose.SkipTest("Datastore not supported")
         p.load('datastore')
+        p.load('datapusher')
         ctd.CreateTestData.create()
         cls.sysadmin_user = model.User.get('testsysadmin')
         cls.normal_user = model.User.get('annafan')
@@ -49,6 +50,7 @@ def setup_class(cls):
     def teardown_class(cls):
         rebuild_all_dbs(cls.Session)
         p.unload('datastore')
+        p.unload('datapusher')
 
     def test_create_requires_auth(self):
         resource = model.Package.get('annakarenina').resources[0]
diff --git a/setup.py b/setup.py
index 276e03a5fdf..eb0d8058d6f 100644
--- a/setup.py
+++ b/setup.py
@@ -115,6 +115,7 @@
     organizations=ckanext.organizations.forms:OrganizationForm
     organizations_dataset=ckanext.organizations.forms:OrganizationDatasetForm
     datastore=ckanext.datastore.plugin:DatastorePlugin
+    datapusher=ckanext.datapusher.plugin:DatapusherPlugin
     test_tag_vocab_plugin=ckanext.test_tag_vocab_plugin:MockVocabTagsPlugin
     resource_proxy=ckanext.resourceproxy.plugin:ResourceProxy
     text_preview=ckanext.textpreview.plugin:TextPreview

From 85c50e18c43cccc69446aa977b082de0411a0560 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Tue, 20 Aug 2013 11:49:31 +0200
Subject: [PATCH 14/87] [#938] Validate datapusher_submit with a schema

---
 ckanext/datapusher/logic/action.py | 10 +++++++---
 ckanext/datapusher/logic/schema.py | 25 +++++++++++++++++++++++++
 ckanext/datapusher/plugin.py       |  3 ++-
 ckanext/datastore/logic/action.py  |  2 +-
 4 files changed, 35 insertions(+), 5 deletions(-)
 create mode 100644 ckanext/datapusher/logic/schema.py

diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py
index 5957887d44e..47a255778f7 100644
--- a/ckanext/datapusher/logic/action.py
+++ b/ckanext/datapusher/logic/action.py
@@ -9,6 +9,7 @@
 import ckan.lib.navl.dictization_functions
 import ckan.logic as logic
 import ckan.plugins as p
+import ckanext.datapusher.logic.schema as dpschema
 
 log = logging.getLogger(__name__)
 _get_or_bust = logic.get_or_bust
@@ -40,9 +41,12 @@ def datapusher_submit(context, data_dict):
     :rtype: boolean
     '''
 
-    if 'id' in data_dict:
-        data_dict['resource_id'] = data_dict['id']
-    res_id = _get_or_bust(data_dict, 'resource_id')
+    schema = context.get('schema', dpschema.datapusher_submit_schema())
+    data_dict, errors = _validate(data_dict, schema, context)
+    if errors:
+        raise p.toolkit.ValidationError(errors)
+
+    res_id = data_dict['resource_id']
 
     p.toolkit.check_access('datapusher_submit', context, data_dict)
 
diff --git a/ckanext/datapusher/logic/schema.py b/ckanext/datapusher/logic/schema.py
new file mode 100644
index 00000000000..07e8a36aec8
--- /dev/null
+++ b/ckanext/datapusher/logic/schema.py
@@ -0,0 +1,25 @@
+import ckan.plugins as p
+import ckanext.datastore.logic.schema as dsschema
+
+get_validator = p.toolkit.get_validator
+
+not_missing = get_validator('not_missing')
+not_empty = get_validator('not_empty')
+resource_id_exists = get_validator('resource_id_exists')
+package_id_exists = get_validator('package_id_exists')
+ignore_missing = get_validator('ignore_missing')
+empty = get_validator('empty')
+boolean_validator = get_validator('boolean_validator')
+int_validator = get_validator('int_validator')
+OneOf = get_validator('OneOf')
+
+
+def datapusher_submit_schema():
+    schema = {
+        'resource_id': [not_missing, not_empty, unicode],
+        'id': [ignore_missing],
+        'set_url_type': [ignore_missing, boolean_validator],
+        '__junk': [empty],
+        '__before': [dsschema.rename('id', 'resource_id')]
+    }
+    return schema
diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py
index 17f9022fd22..650282ea9e1 100644
--- a/ckanext/datapusher/plugin.py
+++ b/ckanext/datapusher/plugin.py
@@ -47,7 +47,8 @@ def notify(self, entity, operation=None):
                     'id': entity.get_package_id()
                 })
                 if (not package['private'] and
-                        entity.format in self.datapusher_formats):
+                        entity.format in self.datapusher_formats and
+                        entity.url_type != 'datapusher'):
                     p.toolkit.get_action('datapusher_submit')(context, {
                         'resource_id': entity.id
                     })
diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
index 9e93deed315..23dae4dc5b0 100644
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -99,7 +99,7 @@ def datastore_create(context, data_dict):
         if has_url:
             p.toolkit.get_action('datapusher_submit')(context, {
                 'resource_id': res['id'],
-                'set_url_to_dump': True
+                'set_url_type': True
             })
         # create empty resource
         else:

From 09bde044490fe73c776064cad2fbb26e41652617 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Tue, 20 Aug 2013 16:02:02 +0200
Subject: [PATCH 15/87] [#1200] Imports and comments clean up

---
 ckanext/datapusher/logic/action.py | 16 ++++++++--------
 ckanext/datastore/logic/action.py  |  9 +++++----
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py
index 47a255778f7..1fe1d5abb8b 100644
--- a/ckanext/datapusher/logic/action.py
+++ b/ckanext/datapusher/logic/action.py
@@ -17,7 +17,7 @@
 
 
 def datapusher_enabled(context, data_dict):
-    """ Returns True if the DataPusher is configured """
+    ''' Returns True if the DataPusher is configured '''
 
     datapusher_url = pylons.config.get('datapusher.url')
     return True if datapusher_url else False
@@ -30,15 +30,15 @@ def datapusher_submit(context, data_dict):
     :param resource_id: The resource id of the resource that the data
         should be imported in. The resource's URL will be used to get the data.
     :type resource_id: string
-    :param set_url_type: If set to true, the ``url_type`` of the resource will
+    :param set_url_type: If set to True, the ``url_type`` of the resource will
         be set to ``datastore`` and the resource URL will automatically point
         to the :ref:`datastore dump <dump>` URL. (optional, default: False)
-    :type set_url_type: boolean
+    :type set_url_type: bool
 
     Returns ``True`` if the job has been submitted and ``False`` if the job
     has not been submitted, i.e. when the datapusher is not configured.
 
-    :rtype: boolean
+    :rtype: bool
     '''
 
     schema = context.get('schema', dpschema.datapusher_submit_schema())
@@ -113,11 +113,11 @@ def datapusher_submit(context, data_dict):
 
 
 def datapusher_hook(context, data_dict):
-    """ Update datapusher task. This action is typically called by the
+    ''' Update datapusher task. This action is typically called by the
     datapusher whenever the status of a job changes.
 
     Expects a job with ``status`` and ``metadata`` with a ``resource_id``.
-    """
+    '''
 
     # TODO: use a schema to validate
 
@@ -147,12 +147,12 @@ def datapusher_hook(context, data_dict):
 
 
 def datapusher_status(context, data_dict):
-    """ Get the status of a datapusher job for a certain resource.
+    ''' Get the status of a datapusher job for a certain resource.
 
     :param resource_id: The resource id of the resource that you want the
         datapusher status for.
     :type resource_id: string
-    """
+    '''
 
     p.toolkit.check_access('datapusher_status', context, data_dict)
 
diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
index 23dae4dc5b0..489616f49dc 100644
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -1,10 +1,6 @@
 import logging
-import json
-import urlparse
-import datetime
 
 import pylons
-import requests
 import sqlalchemy
 
 import ckan.lib.navl.dictization_functions
@@ -101,11 +97,16 @@ def datastore_create(context, data_dict):
                 'resource_id': res['id'],
                 'set_url_type': True
             })
+            # since we'll overwrite the datastore resource anyway, we
+            # don't need to create it here
+            return
         # create empty resource
         else:
             # no need to set the full url because it will be set in before_show
             res['url_type'] = 'datastore'
             p.toolkit.get_action('resource_update')(context, res)
+    else:
+        _check_read_only(context, data_dict)
 
     data_dict['connection_url'] = pylons.config['ckan.datastore.write_url']
 

From d8a9f37bab6d1db3401117690200c9bf7476437a Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Tue, 20 Aug 2013 16:03:13 +0200
Subject: [PATCH 16/87] [#1200] Do not allow writing if url_type != datastore
 unless force=true is passed

---
 ckanext/datastore/logic/action.py      | 35 ++++++++++++++++++++++----
 ckanext/datastore/logic/schema.py      |  3 +++
 ckanext/datastore/tests/helpers.py     | 11 ++++++++
 ckanext/datastore/tests/test_create.py |  4 ++-
 ckanext/datastore/tests/test_delete.py |  4 ++-
 ckanext/datastore/tests/test_dump.py   |  1 +
 ckanext/datastore/tests/test_search.py | 13 +++++++---
 ckanext/datastore/tests/test_upsert.py |  8 +++++-
 8 files changed, 68 insertions(+), 11 deletions(-)

diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
index 489616f49dc..ee2a44e9e4a 100644
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -35,6 +35,8 @@ def datastore_create(context, data_dict):
 
     :param resource_id: resource id that the data is going to be stored against.
     :type resource_id: string
+    :param force: set to True to edit a read-only resource
+    :type force: bool (optional, default: False)
     :param resource: resource dictionary that is passed to
         :meth:`~ckan.logic.action.create.resource_create`.
         Use instead of ``resource_id`` (optional)
@@ -154,6 +156,8 @@ def datastore_upsert(context, data_dict):
 
     :param resource_id: resource id that the data is going to be stored under.
     :type resource_id: string
+    :param force: set to True to edit a read-only resource
+    :type force: bool (optional, default: False)
     :param records: the data, eg: [{"dob": "2005", "some_stuff": ["a","b"]}] (optional)
     :type records: list of dictionaries
     :param method: the method to use to put the data into the datastore.
@@ -174,6 +178,10 @@ def datastore_upsert(context, data_dict):
     if errors:
         raise p.toolkit.ValidationError(errors)
 
+    p.toolkit.check_access('datastore_upsert', context, data_dict)
+
+    _check_read_only(context, data_dict)
+
     data_dict['connection_url'] = pylons.config['ckan.datastore.write_url']
 
     res_id = data_dict['resource_id']
@@ -187,8 +195,6 @@ def datastore_upsert(context, data_dict):
             u'Resource "{0}" was not found.'.format(res_id)
         ))
 
-    p.toolkit.check_access('datastore_upsert', context, data_dict)
-
     result = db.upsert(context, data_dict)
     result.pop('id', None)
     result.pop('connection_url')
@@ -200,6 +206,8 @@ def datastore_delete(context, data_dict):
 
     :param resource_id: resource id that the data will be deleted from. (optional)
     :type resource_id: string
+    :param force: set to True to edit a read-only resource
+    :type force: bool (optional, default: False)
     :param filters: filters to apply before deleting (eg {"name": "fred"}).
                    If missing delete whole table and all dependent views. (optional)
     :type filters: dictionary
@@ -218,6 +226,10 @@ def datastore_delete(context, data_dict):
     if errors:
         raise p.toolkit.ValidationError(errors)
 
+    p.toolkit.check_access('datastore_delete', context, data_dict)
+
+    _check_read_only(context, data_dict)
+
     data_dict['connection_url'] = pylons.config['ckan.datastore.write_url']
 
     res_id = data_dict['resource_id']
@@ -231,8 +243,6 @@ def datastore_delete(context, data_dict):
             u'Resource "{0}" was not found.'.format(res_id)
         ))
 
-    p.toolkit.check_access('datastore_delete', context, data_dict)
-
     result = db.delete(context, data_dict)
     result.pop('id', None)
     result.pop('connection_url')
@@ -431,7 +441,7 @@ def datastore_make_public(context, data_dict):
 
 
 def _resource_exists(context, data_dict):
-    # Returns true if the resource exists in CKAN and in the datastore
+    ''' Returns true if the resource exists in CKAN and in the datastore '''
     model = _get_or_bust(context, 'model')
     res_id = _get_or_bust(data_dict, 'resource_id')
     if not model.Resource.get(res_id):
@@ -441,3 +451,18 @@ def _resource_exists(context, data_dict):
                                         WHERE name = :id AND alias_of IS NULL''')
     results = db._get_engine(data_dict).execute(resources_sql, id=res_id)
     return results.rowcount > 0
+
+
+def _check_read_only(context, data_dict):
+    ''' Raises exception if the resource is read-only.
+    Make sure the resource id is in resource_id
+    '''
+    if data_dict.get('force'):
+        return
+    res = p.toolkit.get_action('resource_show')(
+        context, {'id': data_dict['resource_id']})
+    if res.get('url_type') != 'datastore':
+        raise p.toolkit.ValidationError({
+            'read-only': ['Cannot edit read-only resource. Either pass'
+                          '"force=True" or change url-type to "datastore"']
+        })
diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py
index 018eb249d79..6f0a74723ca 100644
--- a/ckanext/datastore/logic/schema.py
+++ b/ckanext/datastore/logic/schema.py
@@ -68,6 +68,7 @@ def json_validator(value, context):
 def datastore_create_schema():
     schema = {
         'resource_id': [ignore_missing, unicode, resource_id_exists],
+        'force': [ignore_missing, boolean_validator],
         'id': [ignore_missing],
         'aliases': [ignore_missing, list_of_strings_or_string],
         'fields': {
@@ -85,6 +86,7 @@ def datastore_create_schema():
 def datastore_upsert_schema():
     schema = {
         'resource_id': [not_missing, not_empty, unicode],
+        'force': [ignore_missing, boolean_validator],
         'id': [ignore_missing],
         'method': [ignore_missing, unicode, OneOf(
             ['upsert', 'insert', 'update'])],
@@ -97,6 +99,7 @@ def datastore_upsert_schema():
 def datastore_delete_schema():
     schema = {
         'resource_id': [not_missing, not_empty, unicode],
+        'force': [ignore_missing, boolean_validator],
         'id': [ignore_missing],
         '__junk': [empty],
         '__before': [rename('id', 'resource_id')]
diff --git a/ckanext/datastore/tests/helpers.py b/ckanext/datastore/tests/helpers.py
index cf83f57378a..3ee89cdda20 100644
--- a/ckanext/datastore/tests/helpers.py
+++ b/ckanext/datastore/tests/helpers.py
@@ -1,6 +1,8 @@
 import ckan.model as model
 import ckan.lib.cli as cli
 
+import ckan.plugins as p
+
 
 def extract(d, keys):
     return dict((k, d[k]) for k in keys if k in d)
@@ -29,3 +31,12 @@ def rebuild_all_dbs(Session):
         model.repo.tables_created_and_initialised = False
     clear_db(Session)
     model.repo.rebuild_db()
+
+
+def set_url_type(resources, user):
+    context = {'user': user.name}
+    for resource in resources:
+        resource = p.toolkit.get_action('resource_show')(
+            context, {'id': resource.id})
+        resource['url_type'] = 'datastore'
+        p.toolkit.get_action('resource_update')(context, resource)
diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py
index 7a6933008e1..f95e8044951 100644
--- a/ckanext/datastore/tests/test_create.py
+++ b/ckanext/datastore/tests/test_create.py
@@ -17,7 +17,7 @@
 import ckan.config.middleware as middleware
 
 import ckanext.datastore.db as db
-from ckanext.datastore.tests.helpers import rebuild_all_dbs
+from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type
 
 
 # avoid hanging tests https://github.com/gabrielfalcao/HTTPretty/issues/34
@@ -45,6 +45,8 @@ def setup_class(cls):
         engine = db._get_engine(
             {'connection_url': pylons.config['ckan.datastore.write_url']})
         cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
+        set_url_type(
+            model.Package.get('annakarenina').resources, cls.sysadmin_user)
 
     @classmethod
     def teardown_class(cls):
diff --git a/ckanext/datastore/tests/test_delete.py b/ckanext/datastore/tests/test_delete.py
index ce1b02efdb1..bb7db39217a 100644
--- a/ckanext/datastore/tests/test_delete.py
+++ b/ckanext/datastore/tests/test_delete.py
@@ -11,7 +11,7 @@
 import ckan.tests as tests
 
 import ckanext.datastore.db as db
-from ckanext.datastore.tests.helpers import rebuild_all_dbs
+from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type
 
 
 class TestDatastoreDelete(tests.WsgiAppCase):
@@ -43,6 +43,8 @@ def setup_class(cls):
         engine = db._get_engine(
             {'connection_url': pylons.config['ckan.datastore.write_url']})
         cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
+        set_url_type(
+            model.Package.get('annakarenina').resources, cls.sysadmin_user)
 
     @classmethod
     def teardown_class(cls):
diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py
index 041105af986..6061535f4ba 100644
--- a/ckanext/datastore/tests/test_dump.py
+++ b/ckanext/datastore/tests/test_dump.py
@@ -32,6 +32,7 @@ def setup_class(cls):
         resource = model.Package.get('annakarenina').resources[0]
         cls.data = {
             'resource_id': resource.id,
+            'force': True,
             'aliases': 'books',
             'fields': [{'id': u'b\xfck', 'type': 'text'},
                        {'id': 'author', 'type': 'text'},
diff --git a/ckanext/datastore/tests/test_search.py b/ckanext/datastore/tests/test_search.py
index 21cb19de4e2..1b7c9486da7 100644
--- a/ckanext/datastore/tests/test_search.py
+++ b/ckanext/datastore/tests/test_search.py
@@ -30,6 +30,7 @@ def setup_class(cls):
         cls.resource = cls.dataset.resources[0]
         cls.data = {
             'resource_id': cls.resource.id,
+            'force': True,
             'aliases': 'books3',
             'fields': [{'id': u'b\xfck', 'type': 'text'},
                        {'id': 'author', 'type': 'text'},
@@ -116,7 +117,7 @@ def test_search_private_dataset(self):
             context,
             {'name': 'privatedataset',
              'private': True,
-             'owner_org' : self.organization['id'],
+             'owner_org': self.organization['id'],
              'groups': [{
                  'id': group.id
              }]})
@@ -128,6 +129,7 @@ def test_search_private_dataset(self):
 
         postparams = '%s=1' % json.dumps({
             'resource_id': resource['id'],
+            'force': True
         })
         auth = {'Authorization': str(self.sysadmin_user.apikey)}
         res = self.app.post('/api/action/datastore_create', params=postparams,
@@ -425,6 +427,7 @@ def setup_class(cls):
         resource = model.Package.get('annakarenina').resources[0]
         cls.data = dict(
             resource_id=resource.id,
+            force=True,
             fields=[
               {'id': 'id'},
               {'id': 'date', 'type':'date'},
@@ -499,6 +502,7 @@ def setup_class(cls):
         resource = cls.dataset.resources[0]
         cls.data = {
             'resource_id': resource.id,
+            'force': True,
             'aliases': 'books4',
             'fields': [{'id': u'b\xfck', 'type': 'text'},
                        {'id': 'author', 'type': 'text'},
@@ -517,7 +521,7 @@ def setup_class(cls):
                            extra_environ=auth)
         res_dict = json.loads(res.body)
         assert res_dict['success'] is True
-        
+
         # Make an organization, because private datasets must belong to one.
         cls.organization = tests.call_action_api(
             cls.app, 'organization_create',
@@ -669,6 +673,7 @@ def test_new_datastore_table_from_private_resource(self):
 
         postparams = '%s=1' % json.dumps({
             'resource_id': resource['id'],
+            'force': True
         })
         auth = {'Authorization': str(self.sysadmin_user.apikey)}
         res = self.app.post('/api/action/datastore_create', params=postparams,
@@ -708,7 +713,9 @@ def test_making_resource_private_makes_datastore_private(self):
              'package_id': package['id']})
 
         postparams = '%s=1' % json.dumps({
-            'resource_id': resource['id']})
+            'resource_id': resource['id'],
+            'force': True
+        })
         auth = {'Authorization': str(self.sysadmin_user.apikey)}
         res = self.app.post('/api/action/datastore_create', params=postparams,
                             extra_environ=auth)
diff --git a/ckanext/datastore/tests/test_upsert.py b/ckanext/datastore/tests/test_upsert.py
index e27f8b9c523..9d65700198a 100644
--- a/ckanext/datastore/tests/test_upsert.py
+++ b/ckanext/datastore/tests/test_upsert.py
@@ -11,7 +11,7 @@
 import ckan.tests as tests
 
 import ckanext.datastore.db as db
-from ckanext.datastore.tests.helpers import rebuild_all_dbs
+from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type
 
 
 class TestDatastoreUpsert(tests.WsgiAppCase):
@@ -26,6 +26,8 @@ def setup_class(cls):
         ctd.CreateTestData.create()
         cls.sysadmin_user = model.User.get('testsysadmin')
         cls.normal_user = model.User.get('annafan')
+        set_url_type(
+            model.Package.get('annakarenina').resources, cls.sysadmin_user)
         resource = model.Package.get('annakarenina').resources[0]
         cls.data = {
             'resource_id': resource.id,
@@ -249,6 +251,8 @@ def setup_class(cls):
         ctd.CreateTestData.create()
         cls.sysadmin_user = model.User.get('testsysadmin')
         cls.normal_user = model.User.get('annafan')
+        set_url_type(
+            model.Package.get('annakarenina').resources, cls.sysadmin_user)
         resource = model.Package.get('annakarenina').resources[0]
         cls.data = {
             'resource_id': resource.id,
@@ -349,6 +353,8 @@ def setup_class(cls):
         ctd.CreateTestData.create()
         cls.sysadmin_user = model.User.get('testsysadmin')
         cls.normal_user = model.User.get('annafan')
+        set_url_type(
+            model.Package.get('annakarenina').resources, cls.sysadmin_user)
         resource = model.Package.get('annakarenina').resources[0]
         hhguide = u"hitchhiker's guide to the galaxy"
         cls.data = {

From 34b4466358dca5f9a6f27dd693cc32e28a3aa644 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Sun, 25 Aug 2013 10:56:41 +0200
Subject: [PATCH 17/87] [#938] Prepend config option with ckan.

---
 ckan/config/deployment.ini_tmpl    |  4 ++--
 ckanext/datapusher/logic/action.py |  4 ++--
 doc/configuration.rst              | 12 ++++++------
 test-core.ini                      |  2 ++
 4 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl
index 0729c606f3c..665cc8ec08e 100644
--- a/ckan/config/deployment.ini_tmpl
+++ b/ckan/config/deployment.ini_tmpl
@@ -138,8 +138,8 @@ ckan.feeds.author_link =
 
 # Make sure you have set up the DataStore
 
-datapusher.formats = csv
-datapusher.url = http://datapusher.ckan.org/
+ckan.datapusher.formats = csv
+ckan.datapusher.url = http://datapusher.ckan.org/
 
 ## Activity Streams Settings
 
diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py
index 1fe1d5abb8b..c7161a9ccc5 100644
--- a/ckanext/datapusher/logic/action.py
+++ b/ckanext/datapusher/logic/action.py
@@ -53,7 +53,7 @@ def datapusher_submit(context, data_dict):
     if not p.toolkit.get_action('datapusher_enabled')(context):
         return False
 
-    datapusher_url = pylons.config.get('datapusher.url')
+    datapusher_url = pylons.config.get('ckan.datapusher.url')
 
     callback_url = p.toolkit.url_for(
         controller='api', action='action', logic_function='datapusher_hook',
@@ -172,7 +172,7 @@ def datapusher_status(context, data_dict):
         'key': 'job_key'
     })
 
-    datapusher_url = pylons.config.get('datapusher.url')
+    datapusher_url = pylons.config.get('ckan.datapusher.url')
     if not datapusher_url:
         raise p.toolkit.ValidationError(
             {'configuration': ['DataPusher not configured.']})
diff --git a/doc/configuration.rst b/doc/configuration.rst
index 284dbbfde76..88794f2d05a 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -1065,23 +1065,23 @@ Only used with the Google storage backend.
 DataPusher Settings
 -------------------
 
-.. _datapusher.formats:
+.. _ckan.datapusher.formats:
 
-datapusher.formats
+ckan.datapusher.formats
 ^^^^^^^^^^^^^^^^^^
 
 Example::
-  datapusher.formats = csv xls xlsx
+  ckan.datapusher.formats = csv xls xlsx
 
 .. todo:: Expand
 
-.. _datapusher.url
+.. _ckan.datapusher.url
 
-datapusher.url
+ckan.datapusher.url
 ^^^^^^^^^^^^^^
 
 Example::
-  datapusher.url = http://datapusher.ckan.org/
+  ckan.datapusher.url = http://datapusher.ckan.org/
 
 .. todo:: Expand
 
diff --git a/test-core.ini b/test-core.ini
index 21dc08cfae9..da5c1a0814a 100644
--- a/test-core.ini
+++ b/test-core.ini
@@ -25,6 +25,8 @@ sqlalchemy.url = postgresql://ckan_default:pass@localhost/ckan_test
 ckan.datastore.write_url = postgresql://ckan_default:pass@localhost/datastore_test
 ckan.datastore.read_url = postgresql://datastore_default:pass@localhost/datastore_test
 
+ckan.datapusher.url = http://datapusher.ckan.org/
+
 ## Solr support
 solr_url = http://127.0.0.1:8983/solr
 

From 91f6a38ec63f6fe5d31598230eb32a346a39ca9f Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Sun, 25 Aug 2013 10:57:36 +0200
Subject: [PATCH 18/87] [#1196] Move datapusher tests

---
 ckanext/datapusher/tests/__init__.py   |   0
 ckanext/datapusher/tests/test.py       | 192 +++++++++++++++++++++++++
 ckanext/datastore/tests/test_create.py | 140 ------------------
 3 files changed, 192 insertions(+), 140 deletions(-)
 create mode 100644 ckanext/datapusher/tests/__init__.py
 create mode 100644 ckanext/datapusher/tests/test.py

diff --git a/ckanext/datapusher/tests/__init__.py b/ckanext/datapusher/tests/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py
new file mode 100644
index 00000000000..f867274d50d
--- /dev/null
+++ b/ckanext/datapusher/tests/test.py
@@ -0,0 +1,192 @@
+import json
+import httpretty
+import nose
+import sys
+import datetime
+
+import pylons
+from pylons import config
+import sqlalchemy.orm as orm
+import paste.fixture
+
+import ckan.plugins as p
+import ckan.lib.create_test_data as ctd
+import ckan.model as model
+import ckan.tests as tests
+import ckan.config.middleware as middleware
+
+import ckanext.datastore.db as db
+from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type
+
+
+# avoid hanging tests https://github.com/gabrielfalcao/HTTPretty/issues/34
+if sys.version_info < (2, 7, 0):
+    import socket
+    socket.setdefaulttimeout(1)
+
+
+class TestDatastoreCreate(tests.WsgiAppCase):
+    sysadmin_user = None
+    normal_user = None
+
+    @classmethod
+    def setup_class(cls):
+
+        wsgiapp = middleware.make_app(config['global_conf'], **config)
+        cls.app = paste.fixture.TestApp(wsgiapp)
+        if not tests.is_datastore_supported():
+            raise nose.SkipTest("Datastore not supported")
+        p.load('datastore')
+        p.load('datapusher')
+        ctd.CreateTestData.create()
+        cls.sysadmin_user = model.User.get('testsysadmin')
+        cls.normal_user = model.User.get('annafan')
+        engine = db._get_engine(
+            {'connection_url': pylons.config['ckan.datastore.write_url']})
+        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
+        set_url_type(
+            model.Package.get('annakarenina').resources, cls.sysadmin_user)
+
+    @classmethod
+    def teardown_class(cls):
+        rebuild_all_dbs(cls.Session)
+        p.unload('datastore')
+        p.unload('datapusher')
+
+    def test_create_ckan_resource_in_package(self):
+        package = model.Package.get('annakarenina')
+        data = {
+            'resource': {'package_id': package.id}
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=200)
+        res_dict = json.loads(res.body)
+
+        assert 'resource_id' in res_dict['result']
+        assert len(model.Package.get('annakarenina').resources) == 3
+
+        res = tests.call_action_api(
+            self.app, 'resource_show', id=res_dict['result']['resource_id'])
+        assert res['url'] == '/datastore/dump/' + res['id'], res
+
+    @httpretty.activate
+    def test_providing_res_with_url_calls_datapusher_correctly(self):
+        pylons.config['datapusher.url'] = 'http://datapusher.ckan.org'
+        httpretty.HTTPretty.register_uri(
+            httpretty.HTTPretty.POST,
+            'http://datapusher.ckan.org/job',
+            content_type='application/json',
+            body=json.dumps({'job_id': 'foo', 'job_key': 'bar'}))
+
+        package = model.Package.get('annakarenina')
+
+        tests.call_action_api(
+            self.app, 'datastore_create', apikey=self.sysadmin_user.apikey,
+            resource=dict(package_id=package.id, url='demo.ckan.org'))
+
+        assert len(package.resources) == 4, len(package.resources)
+        resource = package.resources[3]
+        data = json.loads(httpretty.last_request().body)
+        assert data['metadata']['resource_id'] == resource.id, data
+        assert data['result_url'].endswith('/action/datapusher_hook'), data
+        assert data['result_url'].startswith('http://'), data
+
+    def test_cant_provide_resource_and_resource_id(self):
+        package = model.Package.get('annakarenina')
+        resource = package.resources[0]
+        data = {
+            'resource_id': resource.id,
+            'resource': {'package_id': package.id}
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['error']['__type'] == 'Validation Error'
+
+    @httpretty.activate
+    def test_send_datapusher_creates_task(self):
+        httpretty.HTTPretty.register_uri(
+            httpretty.HTTPretty.POST,
+            'http://datapusher.ckan.org/job',
+            content_type='application/json',
+            body=json.dumps({'job_id': 'foo', 'job_key': 'bar'}))
+
+        package = model.Package.get('annakarenina')
+        resource = package.resources[0]
+
+        context = {
+            'ignore_auth': True,
+            'user': self.sysadmin_user.name
+        }
+
+        p.toolkit.get_action('datapusher_submit')(context, {
+            'resource_id': resource.id
+        })
+
+        task = p.toolkit.get_action('task_status_show')(context, {
+            'entity_id': resource.id,
+            'task_type': 'datapusher',
+            'key': 'job_id'
+        })
+
+        assert task['state'] == 'pending', task
+
+    def test_datapusher_hook(self):
+        package = model.Package.get('annakarenina')
+        resource = package.resources[0]
+
+        context = {
+            'user': self.sysadmin_user.name
+        }
+
+        p.toolkit.get_action('task_status_update')(context, {
+            'entity_id': resource.id,
+            'entity_type': 'resource',
+            'task_type': 'datapusher',
+            'key': 'job_id',
+            'value': 'my_id',
+            'last_updated': str(datetime.datetime.now()),
+            'state': 'pending'
+        })
+
+        p.toolkit.get_action('task_status_update')(context, {
+            'entity_id': resource.id,
+            'entity_type': 'resource',
+            'task_type': 'datapusher',
+            'key': 'job_key',
+            'value': 'my_key',
+            'last_updated': str(datetime.datetime.now()),
+            'state': 'pending'
+        })
+
+        data = {
+            'status': 'success',
+            'metadata': {
+                'resource_id': resource.id
+            }
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datapusher_hook', params=postparams,
+                            extra_environ=auth, status=200)
+        print res.body
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is True
+
+        task = tests.call_action_api(
+            self.app, 'task_status_show', entity_id=resource.id,
+            task_type='datapusher', key='job_id')
+
+        assert task['state'] == 'success', task
+
+        task = tests.call_action_api(
+            self.app, 'task_status_show', entity_id=resource.id,
+            task_type='datapusher', key='job_key')
+
+        assert task['state'] == 'success', task
\ No newline at end of file
diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py
index f95e8044951..f2db232af8a 100644
--- a/ckanext/datastore/tests/test_create.py
+++ b/ckanext/datastore/tests/test_create.py
@@ -38,7 +38,6 @@ def setup_class(cls):
         if not tests.is_datastore_supported():
             raise nose.SkipTest("Datastore not supported")
         p.load('datastore')
-        p.load('datapusher')
         ctd.CreateTestData.create()
         cls.sysadmin_user = model.User.get('testsysadmin')
         cls.normal_user = model.User.get('annafan')
@@ -52,7 +51,6 @@ def setup_class(cls):
     def teardown_class(cls):
         rebuild_all_dbs(cls.Session)
         p.unload('datastore')
-        p.unload('datapusher')
 
     def test_create_requires_auth(self):
         resource = model.Package.get('annakarenina').resources[0]
@@ -540,144 +538,6 @@ def test_create_basic(self):
 
         assert res_dict['success'] is True, res_dict
 
-    def test_create_ckan_resource_in_package(self):
-        package = model.Package.get('annakarenina')
-        data = {
-            'resource': {'package_id': package.id}
-        }
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, status=200)
-        res_dict = json.loads(res.body)
-
-        assert 'resource_id' in res_dict['result']
-        assert len(model.Package.get('annakarenina').resources) == 3
-
-        res = tests.call_action_api(
-            self.app, 'resource_show', id=res_dict['result']['resource_id'])
-        assert res['url'] == '/datastore/dump/' + res['id'], res
-
-    @httpretty.activate
-    def test_providing_res_with_url_calls_datapusher_correctly(self):
-        pylons.config['datapusher.url'] = 'http://datapusher.ckan.org'
-        httpretty.HTTPretty.register_uri(
-            httpretty.HTTPretty.POST,
-            'http://datapusher.ckan.org/job',
-            content_type='application/json',
-            body=json.dumps({'job_id': 'foo', 'job_key': 'bar'}))
-
-        package = model.Package.get('annakarenina')
-
-        tests.call_action_api(
-            self.app, 'datastore_create', apikey=self.sysadmin_user.apikey,
-            resource=dict(package_id=package.id, url='demo.ckan.org'))
-
-        assert len(package.resources) == 4, len(package.resources)
-        resource = package.resources[3]
-        data = json.loads(httpretty.last_request().body)
-        assert data['metadata']['resource_id'] == resource.id, data
-        assert data['result_url'].endswith('/action/datapusher_hook'), data
-        assert data['result_url'].startswith('http://'), data
-
-    def test_cant_provide_resource_and_resource_id(self):
-        package = model.Package.get('annakarenina')
-        resource = package.resources[0]
-        data = {
-            'resource_id': resource.id,
-            'resource': {'package_id': package.id}
-        }
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['error']['__type'] == 'Validation Error'
-
-    @httpretty.activate
-    def test_send_datapusher_creates_task(self):
-        httpretty.HTTPretty.register_uri(
-            httpretty.HTTPretty.POST,
-            'http://datapusher.ckan.org/job',
-            content_type='application/json',
-            body=json.dumps({'job_id': 'foo', 'job_key': 'bar'}))
-
-        package = model.Package.get('annakarenina')
-        resource = package.resources[0]
-
-        context = {
-            'ignore_auth': True,
-            'user': self.sysadmin_user.name
-        }
-
-        p.toolkit.get_action('datapusher_submit')(context, {
-            'resource_id': resource.id
-        })
-
-        task = p.toolkit.get_action('task_status_show')(context, {
-            'entity_id': resource.id,
-            'task_type': 'datapusher',
-            'key': 'job_id'
-        })
-
-        assert task['state'] == 'pending', task
-
-    def test_datapusher_hook(self):
-        package = model.Package.get('annakarenina')
-        resource = package.resources[0]
-
-        context = {
-            'user': self.sysadmin_user.name
-        }
-
-        p.toolkit.get_action('task_status_update')(context, {
-            'entity_id': resource.id,
-            'entity_type': 'resource',
-            'task_type': 'datapusher',
-            'key': 'job_id',
-            'value': 'my_id',
-            'last_updated': str(datetime.datetime.now()),
-            'state': 'pending'
-        })
-
-        p.toolkit.get_action('task_status_update')(context, {
-            'entity_id': resource.id,
-            'entity_type': 'resource',
-            'task_type': 'datapusher',
-            'key': 'job_key',
-            'value': 'my_key',
-            'last_updated': str(datetime.datetime.now()),
-            'state': 'pending'
-        })
-
-        data = {
-            'status': 'success',
-            'metadata': {
-                'resource_id': resource.id
-            }
-        }
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datapusher_hook', params=postparams,
-                            extra_environ=auth, status=200)
-        print res.body
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is True
-
-        task = tests.call_action_api(
-            self.app, 'task_status_show', entity_id=resource.id,
-            task_type='datapusher', key='job_id')
-
-        assert task['state'] == 'success', task
-
-        task = tests.call_action_api(
-            self.app, 'task_status_show', entity_id=resource.id,
-            task_type='datapusher', key='job_key')
-
-        assert task['state'] == 'success', task
-
     def test_guess_types(self):
         resource = model.Package.get('annakarenina').resources[1]
 

From ef7b3e94c3bc09d60086bbc4ae8b320ad8c61b94 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Sun, 25 Aug 2013 11:13:21 +0200
Subject: [PATCH 19/87] [#1183] Remove datapusher_enabled action and move
 before_show to datstore. Instead of using the action or the template helper,
 check whether the plugin is enabled. Moved the action so that changing the
 url works even if the datapusher is disabled.

---
 ckan/templates/package/resource_read.html     |  4 +++-
 .../templates/snippets/datapusher_status.html | 14 +++++------
 ckanext/datapusher/helpers.py                 |  4 ----
 ckanext/datapusher/logic/action.py            | 10 --------
 ckanext/datapusher/plugin.py                  | 24 +++++++------------
 ckanext/datastore/logic/action.py             | 17 +++++++++----
 ckanext/datastore/plugin.py                   | 13 +++++++++-
 7 files changed, 41 insertions(+), 45 deletions(-)

diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html
index d4433d4a4a8..f914780d179 100644
--- a/ckan/templates/package/resource_read.html
+++ b/ckan/templates/package/resource_read.html
@@ -27,7 +27,9 @@
           <ul>
             {% block resource_actions_inner %}
             {% if h.check_access('package_update', {'id':pkg.id }) %}
-              <li>{% snippet 'snippets/datapusher_status.html', resource=res %}</li>
+              {% if 'datapusher' in g.plugins %}
+                <li>{% snippet 'snippets/datapusher_status.html', resource=res %}</li>
+              {% endif %}
               <li>{% link_for _('Edit'), controller='package', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn', icon='wrench' %}</li>
             {% endif %}
             {% if res.url %}
diff --git a/ckan/templates/snippets/datapusher_status.html b/ckan/templates/snippets/datapusher_status.html
index 863faeee491..3aa8908aef8 100644
--- a/ckan/templates/snippets/datapusher_status.html
+++ b/ckan/templates/snippets/datapusher_status.html
@@ -4,13 +4,11 @@
 
 #}
 {% if resource.datastore_active %}
-	{% if h.datapusher_enabled() %}
-		{% set job = h.datapusher_status(resource.id) %}
-	  {% set title =  _('Datapusher status: {status}.').format(status=job.status) %}
-	  {% if job.status == 'unknown' %}
-	  	<i title="{{ _(title) }}" class="datapusher-status status-{{ job.status }} icon-circle icon-large"></i>
-	  {% else %}
-	  	<a href="{{ job.job_url }}" class="datapusher-status-link" title="{{ _(title) }}"><i class="datapusher-status status-{{ job.status }} icon-circle icon-large"></i></a>
-  	{% endif %}
+  {% set job = h.datapusher_status(resource.id) %}
+  {% set title =  _('Datapusher status: {status}.').format(status=job.status) %}
+  {% if job.status == 'unknown' %}
+    <i title="{{ _(title) }}" class="datapusher-status status-{{ job.status }} icon-circle icon-large"></i>
+  {% else %}
+    <a href="{{ job.job_url }}" class="datapusher-status-link" title="{{ _(title) }}"><i class="datapusher-status status-{{ job.status }} icon-circle icon-large"></i></a>
   {% endif %}
 {% endif %}
diff --git a/ckanext/datapusher/helpers.py b/ckanext/datapusher/helpers.py
index bf06b2e063a..d5291425b28 100644
--- a/ckanext/datapusher/helpers.py
+++ b/ckanext/datapusher/helpers.py
@@ -9,7 +9,3 @@ def datapusher_status(resource_id):
         return {
             'status': 'unknown'
         }
-
-
-def datapusher_enabled():
-    return toolkit.get_action('datapusher_enabled')({})
diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py
index c7161a9ccc5..41d3e063284 100644
--- a/ckanext/datapusher/logic/action.py
+++ b/ckanext/datapusher/logic/action.py
@@ -16,13 +16,6 @@
 _validate = ckan.lib.navl.dictization_functions.validate
 
 
-def datapusher_enabled(context, data_dict):
-    ''' Returns True if the DataPusher is configured '''
-
-    datapusher_url = pylons.config.get('datapusher.url')
-    return True if datapusher_url else False
-
-
 def datapusher_submit(context, data_dict):
     ''' Submit a job to the datapusher. The datapusher is a service that
     imports tabular data into the datastore.
@@ -50,9 +43,6 @@ def datapusher_submit(context, data_dict):
 
     p.toolkit.check_access('datapusher_submit', context, data_dict)
 
-    if not p.toolkit.get_action('datapusher_enabled')(context):
-        return False
-
     datapusher_url = pylons.config.get('ckan.datapusher.url')
 
     callback_url = p.toolkit.url_for(
diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py
index 650282ea9e1..b20f464d789 100644
--- a/ckanext/datapusher/plugin.py
+++ b/ckanext/datapusher/plugin.py
@@ -23,7 +23,6 @@ class DatapusherPlugin(p.SingletonPlugin):
     p.implements(p.IAuthFunctions)
     p.implements(p.IResourceUrlChange)
     p.implements(p.IDomainObjectModification, inherit=True)
-    p.implements(p.IResourceController, inherit=True)
     p.implements(p.ITemplateHelpers)
 
     legacy_mode = False
@@ -32,9 +31,14 @@ class DatapusherPlugin(p.SingletonPlugin):
     def configure(self, config):
         self.config = config
 
-        datapusher_formats = config.get('datapusher.formats', '').split()
+        datapusher_formats = config.get('ckan.datapusher.formats', '').split()
         self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS
 
+        datapusher_url = config.get('ckan.datapusher.url')
+        if not datapusher_url:
+            raise Exception(
+                'Config option `ckan.datapusher.url` has to be set.')
+
     def notify(self, entity, operation=None):
         if isinstance(entity, model.Resource):
             if (operation == model.domain_object.DomainObjectOperation.new
@@ -54,8 +58,7 @@ def notify(self, entity, operation=None):
                     })
 
     def get_actions(self):
-        return {'datapusher_enabled': action.datapusher_enabled,
-                'datapusher_submit': action.datapusher_submit,
+        return {'datapusher_submit': action.datapusher_submit,
                 'datapusher_hook': action.datapusher_hook,
                 'datapusher_status': action.datapusher_status}
 
@@ -63,17 +66,6 @@ def get_auth_functions(self):
         return {'datapusher_submit': auth.datapusher_submit,
                 'datapusher_status': auth.datapusher_status}
 
-    def before_show(self, resource_dict):
-        ''' Modify the resource url of datastore resources so that
-        they link to the datastore dumps.
-        '''
-        if resource_dict['url_type'] == 'datastore':
-            resource_dict['url'] = p.toolkit.url_for(
-                controller='ckanext.datastore.controller:DatastoreController',
-                action='dump', resource_id=resource_dict['id'])
-        return resource_dict
-
     def get_helpers(self):
         return {
-            'datapusher_status': helpers.datapusher_status,
-            'datapusher_enabled': helpers.datapusher_enabled}
+            'datapusher_status': helpers.datapusher_status}
diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
index ee2a44e9e4a..742462ba8ae 100644
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -88,20 +88,27 @@ def datastore_create(context, data_dict):
 
     if 'resource' in data_dict:
         has_url = 'url' in data_dict['resource']
-        data_dict['resource'].setdefault('url', '_tmp')
+        # A datastore only resource does not have a url in the db
+        data_dict['resource'].setdefault('url', '_datastore_only_resource')
         res = p.toolkit.get_action('resource_create')(context,
                                                       data_dict['resource'])
         data_dict['resource_id'] = res['id']
 
         # create resource from file
         if has_url:
-            p.toolkit.get_action('datapusher_submit')(context, {
-                'resource_id': res['id'],
-                'set_url_type': True
-            })
+            try:
+                p.toolkit.get_action('datapusher_submit')(context, {
+                    'resource_id': res['id'],
+                    'set_url_type': True
+                })
+            except KeyError:
+                raise p.toolkit.ValidationError({'resource': [
+                    'The datapusher has to be enabled.']})
+
             # since we'll overwrite the datastore resource anyway, we
             # don't need to create it here
             return
+
         # create empty resource
         else:
             # no need to set the full url because it will be set in before_show
diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py
index 8eb753f8e46..61cdd783d81 100644
--- a/ckanext/datastore/plugin.py
+++ b/ckanext/datastore/plugin.py
@@ -25,6 +25,7 @@ class DatastorePlugin(p.SingletonPlugin):
     p.implements(p.IResourceUrlChange)
     p.implements(p.IDomainObjectModification, inherit=True)
     p.implements(p.IRoutes, inherit=True)
+    p.implements(p.IResourceController, inherit=True)
 
     legacy_mode = False
     resource_show_action = None
@@ -250,4 +251,14 @@ def before_map(self, m):
         m.connect('/datastore/dump/{resource_id}',
                   controller='ckanext.datastore.controller:DatastoreController',
                   action='dump')
-        return m
\ No newline at end of file
+        return m
+
+    def before_show(self, resource_dict):
+        ''' Modify the resource url of datastore resources so that
+        they link to the datastore dumps.
+        '''
+        if resource_dict['url_type'] == 'datastore':
+            resource_dict['url'] = p.toolkit.url_for(
+                controller='ckanext.datastore.controller:DatastoreController',
+                action='dump', resource_id=resource_dict['id'])
+        return resource_dict

From 4e16bc1abc7578ebcfd564482355f7a5d671a493 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Sun, 25 Aug 2013 11:29:54 +0200
Subject: [PATCH 20/87] PEP8

---
 ckanext/datapusher/tests/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py
index f867274d50d..f47f256cef6 100644
--- a/ckanext/datapusher/tests/test.py
+++ b/ckanext/datapusher/tests/test.py
@@ -189,4 +189,4 @@ def test_datapusher_hook(self):
             self.app, 'task_status_show', entity_id=resource.id,
             task_type='datapusher', key='job_key')
 
-        assert task['state'] == 'success', task
\ No newline at end of file
+        assert task['state'] == 'success', task

From 98508d1a67b0ec0a86a1786988f3195d6535f2be Mon Sep 17 00:00:00 2001
From: Sean Hammond <seanhammond@lavabit.com>
Date: Mon, 26 Aug 2013 16:13:48 +0200
Subject: [PATCH 21/87] [#1011] Fix a broken import in the package controller

An exception was not getting caught because of the way this import was
done.
---
 ckan/controllers/package.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py
index c9940cd57ac..e23c6e39fa1 100644
--- a/ckan/controllers/package.py
+++ b/ckan/controllers/package.py
@@ -19,7 +19,7 @@
 import ckan.lib.datapreview as datapreview
 import ckan.lib.plugins
 import ckan.plugins as p
-import lib.render
+import ckan.lib.render
 
 from ckan.common import OrderedDict, _, json, request, c, g, response
 from home import CACHE_PARAMETERS
@@ -366,7 +366,7 @@ def read(self, id, format='html'):
 
         try:
             return render(template, loader_class=loader)
-        except lib.render.TemplateNotFound:
+        except ckan.lib.render.TemplateNotFound:
             msg = _("Viewing {package_type} datasets in {format} format is "
                     "not supported (template file {file} not found).".format(
                     package_type=package_type, format=format, file=template))

From 2d3bcdd7a04b27be135045cf388ef9c983a3c16a Mon Sep 17 00:00:00 2001
From: Sean Hammond <seanhammond@lavabit.com>
Date: Mon, 26 Aug 2013 16:14:47 +0200
Subject: [PATCH 22/87] [#1011] base.py: Explicitly re-raise TemplateNotFound

Functions should not implicitly raise exceptions by failing to catch
exceptions raised by functions that they call, they should explicitly
catch and re-raise the exceptions instead.
---
 ckan/lib/base.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ckan/lib/base.py b/ckan/lib/base.py
index a91a1170135..cf5cf031f76 100644
--- a/ckan/lib/base.py
+++ b/ckan/lib/base.py
@@ -104,7 +104,10 @@ def render_template():
         # we remove it so any bad templates crash and burn
         del globs['url']
 
-        template_path, template_type = render_.template_info(template_name)
+        try:
+            template_path, template_type = render_.template_info(template_name)
+        except render_.TemplateNotFound:
+            raise
 
         # snippets should not pass the context
         # but allow for legacy genshi templates
@@ -202,6 +205,8 @@ def render_template():
         raise ckan.exceptions.CkanUrlException(
             '\nAn Exception has been raised for template %s\n%s' %
             (template_name, e.message))
+    except render_.TemplateNotFound:
+        raise
 
 
 class ValidationException(Exception):

From c136aac9c65fcde7c03a2e1c7a9e88e4b5b2f595 Mon Sep 17 00:00:00 2001
From: Sean Hammond <seanhammond@lavabit.com>
Date: Mon, 26 Aug 2013 16:16:03 +0200
Subject: [PATCH 23/87] [#1011] Update some exception handling in template.py

Catch the new ckan.lib.render.TemplateNotFound exception that render()
now raises, not the Genshi exception it used to.
---
 ckan/controllers/template.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/ckan/controllers/template.py b/ckan/controllers/template.py
index 1ebc700345b..0755b1a2cf5 100644
--- a/ckan/controllers/template.py
+++ b/ckan/controllers/template.py
@@ -1,6 +1,5 @@
-from genshi.template.loader import TemplateNotFound
-
 import ckan.lib.base as base
+import ckan.lib.render
 
 
 class TemplateController(base.BaseController):
@@ -29,11 +28,11 @@ def view(self, url):
         """
         try:
             return base.render(url)
-        except TemplateNotFound:
+        except ckan.lib.render.TemplateNotFound:
             if url.endswith('.html'):
                 base.abort(404)
             url += '.html'
             try:
                 return base.render(url)
-            except TemplateNotFound:
+            except ckan.lib.render.TemplateNotFound:
                 base.abort(404)

From 52c93774c83b13cbfa98107cba87aed6ad9d1b13 Mon Sep 17 00:00:00 2001
From: joetsoi <joe.yeung.tsoi@gmail.com>
Date: Wed, 4 Sep 2013 17:00:05 +0100
Subject: [PATCH 24/87] [#1224] propogate the NotAuthorized exception extra
 messages to api

---
 ckan/controllers/api.py | 35 +++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/ckan/controllers/api.py b/ckan/controllers/api.py
index 22ea7d4f5c3..795488ee02a 100644
--- a/ckan/controllers/api.py
+++ b/ckan/controllers/api.py
@@ -113,8 +113,10 @@ def _finish_ok(self, response_data=None,
 
         return self._finish(status_int, response_data, content_type)
 
-    def _finish_not_authz(self):
+    def _finish_not_authz(self, extra_msg=None):
         response_data = _('Access denied')
+        if extra_msg:
+            response_data = '%s - %s' % (response_data, extra_msg)
         return self._finish(403, response_data, 'json')
 
     def _finish_not_found(self, extra_msg=None):
@@ -193,10 +195,14 @@ def action(self, logic_function, ver=None):
                                     'data': request_data}
             return_dict['success'] = False
             return self._finish(400, return_dict, content_type='json')
-        except NotAuthorized:
+        except NotAuthorized, e:
             return_dict['error'] = {'__type': 'Authorization Error',
                                     'message': _('Access denied')}
             return_dict['success'] = False
+            
+            if e.extra_msg:
+                return_dict['error']['message'] += ': %s' % e.extra_msg
+
             return self._finish(403, return_dict, content_type='json')
         except NotFound, e:
             return_dict['error'] = {'__type': 'Not Found Error',
@@ -275,8 +281,9 @@ def list(self, ver=None, register=None, subregister=None, id=None):
         except NotFound, e:
             extra_msg = e.extra_msg
             return self._finish_not_found(extra_msg)
-        except NotAuthorized:
-            return self._finish_not_authz()
+        except NotAuthorized, e:
+            extra_msg = e.extra_msg
+            return self._finish_not_authz(extra_msg)
 
     def show(self, ver=None, register=None, subregister=None,
              id=None, id2=None):
@@ -306,8 +313,9 @@ def show(self, ver=None, register=None, subregister=None,
         except NotFound, e:
             extra_msg = e.extra_msg
             return self._finish_not_found(extra_msg)
-        except NotAuthorized:
-            return self._finish_not_authz()
+        except NotAuthorized, e:
+            extra_msg = e.extra_msg
+            return self._finish_not_authz(extra_msg)
 
     def _represent_package(self, package):
         return package.as_dict(ref_package_by=self.ref_package_by,
@@ -352,8 +360,9 @@ def create(self, ver=None, register=None, subregister=None,
                                           data_dict.get("id")))
             return self._finish_ok(response_data,
                                    resource_location=location)
-        except NotAuthorized:
-            return self._finish_not_authz()
+        except NotAuthorized, e:
+            extra_msg = e.extra_msg
+            return self._finish_not_authz(extra_msg)
         except NotFound, e:
             extra_msg = e.extra_msg
             return self._finish_not_found(extra_msg)
@@ -408,8 +417,9 @@ def update(self, ver=None, register=None, subregister=None,
         try:
             response_data = action(context, data_dict)
             return self._finish_ok(response_data)
-        except NotAuthorized:
-            return self._finish_not_authz()
+        except NotAuthorized, e:
+            extra_msg = e.extra_msg
+            return self._finish_not_authz(extra_msg)
         except NotFound, e:
             extra_msg = e.extra_msg
             return self._finish_not_found(extra_msg)
@@ -456,8 +466,9 @@ def delete(self, ver=None, register=None, subregister=None,
         try:
             response_data = action(context, data_dict)
             return self._finish_ok(response_data)
-        except NotAuthorized:
-            return self._finish_not_authz()
+        except NotAuthorized, e:
+            extra_msg = e.extra_msg
+            return self._finish_not_authz(extra_msg)
         except NotFound, e:
             extra_msg = e.extra_msg
             return self._finish_not_found(extra_msg)

From 910ca29a902b265e5788864f533a70d8626cbcf1 Mon Sep 17 00:00:00 2001
From: Nigel Babu <nigelbabu@gmail.com>
Date: Thu, 5 Sep 2013 14:59:14 +0530
Subject: [PATCH 25/87] New option to disable user creation via web

---
 ckan/config/deployment.ini_tmpl |  1 +
 ckan/logic/auth/create.py       | 22 +++++++++++++++-------
 ckan/new_authz.py               |  2 ++
 ckan/templates/header.html      |  4 +++-
 ckan/templates/user/login.html  | 20 +++++++++++---------
 test-core.ini                   |  3 ++-
 6 files changed, 34 insertions(+), 18 deletions(-)

diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl
index 97c4f9412ba..6a0833d2d9f 100644
--- a/ckan/config/deployment.ini_tmpl
+++ b/ckan/config/deployment.ini_tmpl
@@ -66,6 +66,7 @@ ckan.auth.user_create_organizations = true
 ckan.auth.user_delete_groups = true
 ckan.auth.user_delete_organizations = true
 ckan.auth.create_user_via_api = false
+ckan.auth.create_user_via_web = true
 
 
 ## Search Settings
diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py
index e96b789bcf5..35a9f39d876 100644
--- a/ckan/logic/auth/create.py
+++ b/ckan/logic/auth/create.py
@@ -103,14 +103,22 @@ def rating_create(context, data_dict):
     # No authz check in the logic function
     return {'success': True}
 
-def user_create(context, data_dict=None):
-    user = context['user']
 
-    if ('api_version' in context
-            and not new_authz.check_config_permission('create_user_via_api')):
-        return {'success': False, 'msg': _('User %s not authorized to create users') % user}
-    else:
-        return {'success': True}
+def user_create(context, data_dict=None):
+    # create_user_via_api is deprecated
+    using_api = 'api_version' in context
+    create_user_via_api = new_authz.check_config_permission(
+            'create_user_via_api')
+    create_user_via_web = new_authz.check_config_permission(
+            'create_user_via_web')
+
+    if using_api and not create_user_via_api:
+        return {'success': False, 'msg': _('User {user} not authorized to '
+            'create users via the API').format(user=context.get('user'))}
+    if not using_api and not create_user_via_web:
+        return {'success': False, 'msg': _('Not authorized to '
+            'create users')}
+    return {'success': True}
 
 
 def _check_group_auth(context, data_dict):
diff --git a/ckan/new_authz.py b/ckan/new_authz.py
index 00a7dfe0e17..03ca209faf4 100644
--- a/ckan/new_authz.py
+++ b/ckan/new_authz.py
@@ -86,6 +86,7 @@ def _build(self):
 
 def clear_auth_functions_cache():
     _AuthFunctions.clear()
+    CONFIG_PERMISSIONS.clear()
 
 
 def auth_functions_list():
@@ -319,6 +320,7 @@ def get_user_id_for_username(user_name, allow_none=False):
     'user_delete_groups': True,
     'user_delete_organizations': True,
     'create_user_via_api': False,
+    'create_user_via_web': True,
 }
 
 CONFIG_PERMISSIONS = {}
diff --git a/ckan/templates/header.html b/ckan/templates/header.html
index 5b4ebe225a7..268b370e968 100644
--- a/ckan/templates/header.html
+++ b/ckan/templates/header.html
@@ -49,7 +49,9 @@
           <ul class="unstyled">
             {% block header_account_notlogged %}
             <li>{% link_for _('Log in'), controller='user', action='login' %}</li>
-            <li>{% link_for _('Register'), controller='user', action='register', class_='sub' %}</li>
+            {% if h.check_access('user_create') %}
+              <li>{% link_for _('Register'), controller='user', action='register', class_='sub' %}</li>
+            {% endif %}
             {% endblock %}
           </ul>
         </nav>
diff --git a/ckan/templates/user/login.html b/ckan/templates/user/login.html
index c84e3522748..98c99510570 100644
--- a/ckan/templates/user/login.html
+++ b/ckan/templates/user/login.html
@@ -18,15 +18,17 @@ <h1 class="page-heading">{% block page_heading %}{{ _('Login') }}{% endblock %}<
 {% endblock %}
 
 {% block secondary_content %}
-  <section class="module module-narrow module-shallow">
-    <h2 class="module-heading">{{ _('Need an Account?') }}</h2>
-    <div class="module-content">
-      <p>{% trans %}Then sign right up, it only takes a minute.{% endtrans %}</p>
-      <p class="action">
-        <a class="btn" href="{{ h.url_for(controller='user', action='register') }}">{{ _('Create an Account') }}</a>
-      </p>
-    </div>
-  </section>
+  {% if h.check_access('user_create') %}
+    <section class="module module-narrow module-shallow">
+      <h2 class="module-heading">{{ _('Need an Account?') }}</h2>
+      <div class="module-content">
+        <p>{% trans %}Then sign right up, it only takes a minute.{% endtrans %}</p>
+        <p class="action">
+          <a class="btn" href="{{ h.url_for(controller='user', action='register') }}">{{ _('Create an Account') }}</a>
+        </p>
+      </div>
+    </section>
+  {% endif %}
 
   <section class="module module-narrow module-shallow">
     <h2 class="module-heading">{{ _('Forgotten your details?') }}</h2>
diff --git a/test-core.ini b/test-core.ini
index 21dc08cfae9..08aba34ed24 100644
--- a/test-core.ini
+++ b/test-core.ini
@@ -31,6 +31,7 @@ solr_url = http://127.0.0.1:8983/solr
 ckan.auth.user_create_organizations = true
 ckan.auth.user_create_groups = true
 ckan.auth.create_user_via_api = false
+ckan.auth.create_user_via_web = true
 ckan.auth.create_dataset_if_not_in_organization = true
 ckan.auth.anon_create_dataset = false
 ckan.auth.user_delete_groups=true
@@ -80,7 +81,7 @@ smtp.mail_from = info@test.ckan.net
 
 ckan.locale_default = en
 ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv
-ckan.locales_filtered_out = 
+ckan.locales_filtered_out =
 
 ckan.datastore.enabled = 1
 

From 6bc0c3ee3b9ea158ad358c5697e26eedd372ba9f Mon Sep 17 00:00:00 2001
From: Nigel Babu <nigelbabu@gmail.com>
Date: Thu, 5 Sep 2013 14:59:50 +0530
Subject: [PATCH 26/87] Add documentation for new config option

---
 doc/configuration.rst | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/doc/configuration.rst b/doc/configuration.rst
index 0efe9573226..853ffd23939 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -355,6 +355,20 @@ Default value: ``False``
 
 Allow new user accounts to be created via the API.
 
+.. _ckan.auth.create_user_via_web:
+
+ckan.auth.create_user_via_api
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Example::
+
+ ckan.auth.create_user_via_web = True
+
+Default value: ``True``
+
+
+Allow new user accounts to be created via the Web.
+
 .. end_config-authorization
 
 

From c75e487ff99765418d5c82e9f683e2d694afc98d Mon Sep 17 00:00:00 2001
From: Nigel Babu <nigelbabu@gmail.com>
Date: Thu, 5 Sep 2013 16:46:10 +0530
Subject: [PATCH 27/87] Fix docs, remove deprecated comment

---
 ckan/logic/auth/create.py | 1 -
 doc/configuration.rst     | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py
index 35a9f39d876..ec38483bd0d 100644
--- a/ckan/logic/auth/create.py
+++ b/ckan/logic/auth/create.py
@@ -105,7 +105,6 @@ def rating_create(context, data_dict):
 
 
 def user_create(context, data_dict=None):
-    # create_user_via_api is deprecated
     using_api = 'api_version' in context
     create_user_via_api = new_authz.check_config_permission(
             'create_user_via_api')
diff --git a/doc/configuration.rst b/doc/configuration.rst
index 853ffd23939..29ecbd310d8 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -357,7 +357,7 @@ Allow new user accounts to be created via the API.
 
 .. _ckan.auth.create_user_via_web:
 
-ckan.auth.create_user_via_api
+ckan.auth.create_user_via_web
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Example::

From e079385c05876834fad681261b0e68df6c9b9257 Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Tue, 10 Sep 2013 22:36:32 +0100
Subject: [PATCH 28/87] [1200] add defaults and make sure datapusher down does
 not cause errors

---
 ckan/model/meta.py                 |  2 +-
 ckan/plugins/interfaces.py         |  1 -
 ckanext/datapusher/logic/action.py |  1 +
 ckanext/datapusher/plugin.py       | 23 +++++++++++++++--------
 4 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/ckan/model/meta.py b/ckan/model/meta.py
index 795094965c5..0103019bb3a 100644
--- a/ckan/model/meta.py
+++ b/ckan/model/meta.py
@@ -163,4 +163,4 @@ def engine_is_pg(sa_engine=None):
     # Returns true iff the engine is connected to a postgresql database.
     # According to http://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql
     # all Postgres driver names start with `postgresql`
-    return (sa_engine or engine).url.drivername.startswith('postgresql')
+    return (sa_engine or engine).url.drivername.startswith('postgres')
diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py
index aa08fc64bab..d1372d2e895 100644
--- a/ckan/plugins/interfaces.py
+++ b/ckan/plugins/interfaces.py
@@ -443,7 +443,6 @@ def before_view(self, pkg_dict):
 class IResourceController(Interface):
     """
     Hook into the resource controller.
-    (see IGroupController)
     """
 
     def before_show(self, resource_dict):
diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py
index 41d3e063284..3201d35572a 100644
--- a/ckanext/datapusher/logic/action.py
+++ b/ckanext/datapusher/logic/action.py
@@ -50,6 +50,7 @@ def datapusher_submit(context, data_dict):
         ver=3, qualified=True)
 
     user = p.toolkit.get_action('user_show')(context, {'id': context['user']})
+
     try:
         r = requests.post(
             urlparse.urljoin(datapusher_url, 'job'),
diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py
index b20f464d789..04a27db435b 100644
--- a/ckanext/datapusher/plugin.py
+++ b/ckanext/datapusher/plugin.py
@@ -10,7 +10,7 @@
 log = logging.getLogger(__name__)
 _get_or_bust = logic.get_or_bust
 
-DEFAULT_FORMATS = []
+DEFAULT_FORMATS = ['csv', 'xls', 'application/csv', 'application/vnd.ms-excel']
 
 
 class DatastoreException(Exception):
@@ -31,7 +31,7 @@ class DatapusherPlugin(p.SingletonPlugin):
     def configure(self, config):
         self.config = config
 
-        datapusher_formats = config.get('ckan.datapusher.formats', '').split()
+        datapusher_formats = config.get('ckan.datapusher.formats', '').lower().split()
         self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS
 
         datapusher_url = config.get('ckan.datapusher.url')
@@ -46,16 +46,23 @@ def notify(self, entity, operation=None):
                 # if operation is None, resource URL has been changed, as
                 # the notify function in IResourceUrlChange only takes
                 # 1 parameter
-                context = {'model': model, 'ignore_auth': True}
+                context = {'model': model, 'ignore_auth': True, 'defer_commit': True}
                 package = p.toolkit.get_action('package_show')(context, {
                     'id': entity.get_package_id()
                 })
-                if (not package['private'] and
-                        entity.format in self.datapusher_formats and
+                if (not package['private'] and entity.format and
+                        entity.format.lower() in self.datapusher_formats and
                         entity.url_type != 'datapusher'):
-                    p.toolkit.get_action('datapusher_submit')(context, {
-                        'resource_id': entity.id
-                    })
+                    try:
+                        p.toolkit.get_action('datapusher_submit')(context, {
+                            'resource_id': entity.id
+                        })
+                    except p.toolkit.ValidationError, e:
+                        # If datapusher is offline want to catch error instead of
+                        # raising otherwise resource save will fail with 500
+                        log.critical(e)
+                        pass
+
 
     def get_actions(self):
         return {'datapusher_submit': action.datapusher_submit,

From 55ed1ef2b6d4eca22c324ddf627eb7fe1e0a4d24 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Thu, 12 Sep 2013 18:39:42 +0100
Subject: [PATCH 29/87] [#1241] Raise CkanVersionException properly

---
 ckan/plugins/toolkit.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py
index 4e4ac5da2d0..34e9d671cf1 100644
--- a/ckan/plugins/toolkit.py
+++ b/ckan/plugins/toolkit.py
@@ -348,7 +348,7 @@ def _requires_ckan_version(cls, min_version, max_version=None):
             else:
                 error = 'Requires ckan version between %s and %s' % \
                             (min_version, max_version)
-            raise cls.CkanVersionException(error)
+            raise CkanVersionException(error)
 
     def __getattr__(self, name):
         ''' return the function/object requested '''

From b8f449e83555ec850a1f0c4fc4bcbb7ded7b3d04 Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Fri, 20 Sep 2013 05:14:46 +0100
Subject: [PATCH 30/87] [#1200] make resource data page for reuploading and
 make error messages better

---
 ckan/config/routing.py                    |   2 +-
 ckan/logic/schema.py                      |   1 +
 ckan/templates/package/new_resource.html  |  12 +-
 ckan/templates/package/resource_data.html |  45 ++++++++
 ckan/templates/package/resource_read.html |   3 -
 ckanext/datapusher/logic/action.py        | 127 +++++++++++++---------
 ckanext/datapusher/plugin.py              |  56 ++++++++++
 ckanext/datastore/controller.py           |   1 +
 8 files changed, 189 insertions(+), 58 deletions(-)
 create mode 100644 ckan/templates/package/resource_data.html

diff --git a/ckan/config/routing.py b/ckan/config/routing.py
index 0fbb02bb886..6b41d8a66f2 100644
--- a/ckan/config/routing.py
+++ b/ckan/config/routing.py
@@ -246,7 +246,7 @@ def make_map():
                   action='resource_read')
         m.connect('/dataset/{id}/resource_delete/{resource_id}',
                   action='resource_delete')
-        m.connect('/dataset/{id}/resource_edit/{resource_id}',
+        m.connect('resource_edit', '/dataset/{id}/resource_edit/{resource_id}',
                   action='resource_edit')
         m.connect('/dataset/{id}/resource/{resource_id}/download',
                   action='resource_download')
diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py
index f8fe8df7fbd..8058e10aea0 100644
--- a/ckan/logic/schema.py
+++ b/ckan/logic/schema.py
@@ -85,6 +85,7 @@ def default_resource_schema():
         'cache_last_updated': [ignore_missing, isodate],
         'webstore_last_updated': [ignore_missing, isodate],
         'tracking_summary': [ignore_missing],
+        'datastore_active': [ignore],
         '__extras': [ignore_missing, extras_unicode_convert, keep_extras],
     }
 
diff --git a/ckan/templates/package/new_resource.html b/ckan/templates/package/new_resource.html
index 40d8ed97c9e..7eedbad761f 100644
--- a/ckan/templates/package/new_resource.html
+++ b/ckan/templates/package/new_resource.html
@@ -29,8 +29,16 @@ <h2 class="module-heading"><i class="icon-info-sign"></i> {{ _('What\'s a resour
 {% endblock %}
 
 {% block primary_content %}
-  {% if pkg_dict and pkg_dict.state != 'draft' %}
-    <header class="module-content page-header"></header>
+  {% set res = c.resource %}
+  {% if pkg_dict and pkg_dict.state != 'draft' and res %}
+    <header class="module-content page-header">
+      <ul class="nav nav-tabs">
+      {{ h.build_nav_icon('resource_edit', _('Edit Resource'), id=pkg_dict.id, resource_id=res.id) }}
+      {% if 'datapusher' in g.plugins %}
+          {{ h.build_nav_icon('resource_data', _('Resource Data'), id=pkg_dict.id, resource_id=res.id) }}
+      {% endif %}
+      </ul>
+    </header>
   {% endif %}
   {{ super() }}
 {% endblock %}
diff --git a/ckan/templates/package/resource_data.html b/ckan/templates/package/resource_data.html
new file mode 100644
index 00000000000..30c9fe59e98
--- /dev/null
+++ b/ckan/templates/package/resource_data.html
@@ -0,0 +1,45 @@
+{% extends "package/new_resource.html" %}
+
+{% set pkg_dict = c.pkg_dict %}
+{% set res = c.resource %}
+
+{% block subtitle %}{{ h.dataset_display_name(pkg_dict) }} - {{ h.resource_display_name(res) }}{% endblock %}
+
+{% block breadcrumb_content_selected %}{% endblock %}
+
+{% block breadcrumb_content %}
+  {{ super() }}
+  <li class="active"><a href="">{{ _('Edit') }}</a></li>
+{% endblock %}
+
+{% block content_action %}
+  {% link_for _('View resource'), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id, class_='btn', icon='eye-open' %}
+{% endblock %}
+
+{# logged_in is defined in new_resource.html #}
+{% block form %}
+
+   {% set action = h.url_for(controller='ckanext.datapusher.plugin:ResourceDataController', action='resource_data', id=pkg_dict.id, resource_id=res.id) %}
+
+   <form class="dataset-form dataset-resource-form form-horizontal" method="post" action="{{ action }}" >
+      <h3> {{ _('Last Upload Status:') }} {% if status.status %} {{ status.status.capitalize() }} {% else %} {{ _('Not Uploaded Yet') }}  {% endif %} </h3>
+      {% if status.error and status.error.message %}  
+        <h3>  {{ _('Upload Error:') }} {{ status.error.message }} </h3>
+      {% endif %} 
+      {% if status.status %}  
+        <h3>   {{ _('Last Updated: ') }} {{ h.time_ago_from_timestamp(status.last_updated) }} </h3>
+      {% endif %} 
+
+      <button class="btn" name="save" value="go-dataset" type="submit">{{ _('Upload Data') }}</button>
+      {% if status.status and status.task_info %}  
+        <h3>   {{ _('Upload Log ') }} </h3>
+        {{ h.debug_inspect(status.task_info.logs) }}
+      {% endif %} 
+      {{ status }}
+   </form>
+
+{% endblock %}
+
+{% block secondary_content %}
+  {% snippet 'package/snippets/info.html', pkg=pkg_dict, active=pkg_dict.id, action='resource_edit' %}
+{% endblock %}
diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html
index f914780d179..46406746a6f 100644
--- a/ckan/templates/package/resource_read.html
+++ b/ckan/templates/package/resource_read.html
@@ -27,9 +27,6 @@
           <ul>
             {% block resource_actions_inner %}
             {% if h.check_access('package_update', {'id':pkg.id }) %}
-              {% if 'datapusher' in g.plugins %}
-                <li>{% snippet 'snippets/datapusher_status.html', resource=res %}</li>
-              {% endif %}
               <li>{% link_for _('Edit'), controller='package', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn', icon='wrench' %}</li>
             {% endif %}
             {% if res.url %}
diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py
index 3201d35572a..653dd44e0b5 100644
--- a/ckanext/datapusher/logic/action.py
+++ b/ckanext/datapusher/logic/action.py
@@ -51,6 +51,29 @@ def datapusher_submit(context, data_dict):
 
     user = p.toolkit.get_action('user_show')(context, {'id': context['user']})
 
+    task = {
+        'entity_id': res_id,
+        'entity_type': 'resource',
+        'task_type': 'datapusher',
+        'last_updated': str(datetime.datetime.now()),
+        'state': 'submitting',
+        'key': 'datapusher',
+        'value': '{}',
+        'error': '{}',
+    }
+    try:
+        task_id = p.toolkit.get_action('task_status_show')(context, {
+            'entity_id': res_id,
+            'task_type': 'datapusher',
+            'key': 'datapusher'
+        })['id']
+        task['id'] = task_id
+    except logic.NotFound:
+        pass
+
+    result = p.toolkit.get_action('task_status_update')(context, task)
+    task_id = result['id']
+
     try:
         r = requests.post(
             urlparse.urljoin(datapusher_url, 'job'),
@@ -69,36 +92,37 @@ def datapusher_submit(context, data_dict):
             }))
         r.raise_for_status()
     except requests.exceptions.ConnectionError, e:
-        raise p.toolkit.ValidationError({'datapusher': {
-            'message': 'Could not connect to DataPusher.',
-            'details': str(e)}})
+        error = {'message': 'Could not connect to DataPusher.',
+                 'details': str(e)}
+        task['error'] = json.dumps(error)
+        task['state'] = 'error'
+        task['last_updated'] = str(datetime.datetime.now()),
+        p.toolkit.get_action('task_status_update')(context, task)
+        raise p.toolkit.ValidationError(error)
+
     except requests.exceptions.HTTPError, e:
         m = 'An Error occurred while sending the job: {0}'.format(e.message)
         try:
             body = e.response.json()
         except ValueError:
             body = e.response.text
-        raise p.toolkit.ValidationError({'datapusher': {
-            'message': m,
-            'details': body,
-            'status_code': r.status_code}})
-
-    empty_task = {
-        'entity_id': res_id,
-        'entity_type': 'resource',
-        'task_type': 'datapusher',
-        'last_updated': str(datetime.datetime.now()),
-        'state': 'pending'
-    }
-
-    tasks = []
-    for (k, v) in [('job_id', r.json()['job_id']),
-                   ('job_key', r.json()['job_key'])]:
-        t = empty_task.copy()
-        t['key'] = k
-        t['value'] = v
-        tasks.append(t)
-    p.toolkit.get_action('task_status_update_many')(context, {'data': tasks})
+        error = {'message': m,
+                 'details': body,
+                 'status_code': r.status_code}
+        task['error'] = json.dumps(error)
+        task['state'] = 'error'
+        task['last_updated'] = str(datetime.datetime.now()),
+        p.toolkit.get_action('task_status_update')(context, task)
+        raise p.toolkit.ValidationError(error)
+
+    value = json.dumps(
+            {'job_id': r.json()['job_id'],
+             'job_key': r.json()['job_key']}
+            )
+    task['value'] = value
+    task['state'] = 'pending'
+    task['last_updated'] = str(datetime.datetime.now()),
+    p.toolkit.get_action('task_status_update')(context, task)
 
     return True
 
@@ -111,30 +135,20 @@ def datapusher_hook(context, data_dict):
     '''
 
     # TODO: use a schema to validate
-
     p.toolkit.check_access('datapusher_submit', context, data_dict)
 
     res_id = data_dict['metadata']['resource_id']
 
-    task_id = p.toolkit.get_action('task_status_show')(context, {
+    task = p.toolkit.get_action('task_status_show')(context, {
         'entity_id': res_id,
         'task_type': 'datapusher',
-        'key': 'job_id'
+        'key': 'datapusher'
     })
 
-    task_key = p.toolkit.get_action('task_status_show')(context, {
-        'entity_id': res_id,
-        'task_type': 'datapusher',
-        'key': 'job_key'
-    })
-
-    tasks = [task_id, task_key]
-
-    for task in tasks:
-        task['state'] = data_dict['status']
-        task['last_updated'] = str(datetime.datetime.now())
+    task['state'] = data_dict['status']
+    task['last_updated'] = str(datetime.datetime.now())
 
-    p.toolkit.get_action('task_status_update_many')(context, {'data': tasks})
+    p.toolkit.get_action('task_status_update')(context, task)
 
 
 def datapusher_status(context, data_dict):
@@ -151,16 +165,10 @@ def datapusher_status(context, data_dict):
         data_dict['resource_id'] = data_dict['id']
     res_id = _get_or_bust(data_dict, 'resource_id')
 
-    task_id = p.toolkit.get_action('task_status_show')(context, {
+    task = p.toolkit.get_action('task_status_show')(context, {
         'entity_id': res_id,
         'task_type': 'datapusher',
-        'key': 'job_id'
-    })
-
-    task_key = p.toolkit.get_action('task_status_show')(context, {
-        'entity_id': res_id,
-        'task_type': 'datapusher',
-        'key': 'job_key'
+        'key': 'datapusher'
     })
 
     datapusher_url = pylons.config.get('ckan.datapusher.url')
@@ -168,11 +176,26 @@ def datapusher_status(context, data_dict):
         raise p.toolkit.ValidationError(
             {'configuration': ['DataPusher not configured.']})
 
-    url = urlparse.urljoin(datapusher_url, 'job' + '/' + task_id['value'])
+    value = json.loads(task['value'])
+    job_key = value.get('job_key')
+    job_id = value.get('job_id')
+    url = None
+    job_detail = None
+    if job_id:
+        url = urlparse.urljoin(datapusher_url, 'job' + '/' + job_id)
+        try:
+            r = requests.get(url, headers={'Content-Type': 'application/json',
+                                           'Authorization': job_key})
+            job_detail = r.json()
+        except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e:
+            job_detail = {'error': 'cannot connect to datapusher'}
+
     return {
-        'status': task_id['state'],
-        'job_id': task_id['value'],
+        'status': task['state'],
+        'job_id': job_id,
         'job_url': url,
-        'last_updated': task_id['last_updated'],
-        'job_key': task_key['value']
+        'last_updated': task['last_updated'],
+        'job_key': job_key,
+        'task_info': job_detail,
+        'error': json.loads(task['error'])
     }
diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py
index 04a27db435b..a545323a7dd 100644
--- a/ckanext/datapusher/plugin.py
+++ b/ckanext/datapusher/plugin.py
@@ -1,11 +1,14 @@
 import logging
 
 import ckan.plugins as p
+import ckan.lib.base as base
+import ckan.lib.helpers as core_helpers
 import ckanext.datapusher.logic.action as action
 import ckanext.datapusher.logic.auth as auth
 import ckanext.datapusher.helpers as helpers
 import ckan.logic as logic
 import ckan.model as model
+from ckan.common import c, _, request
 
 log = logging.getLogger(__name__)
 _get_or_bust = logic.get_or_bust
@@ -16,6 +19,48 @@
 class DatastoreException(Exception):
     pass
 
+class ResourceDataController(base.BaseController):
+
+    def resource_data(self, id, resource_id):
+
+        if request.method == 'POST':
+            result = request.POST
+            try:
+                c.pkg_dict = p.toolkit.get_action('datapusher_submit')(
+                    None, {'resource_id': resource_id}
+                )
+            except logic.ValidationError:
+                pass
+
+            base.redirect(core_helpers.url_for(
+                 controller='ckanext.datapusher.plugin:ResourceDataController',
+                 action='resource_data',
+                 id=id,
+                 resource_id=resource_id)
+            )
+
+        try:
+            c.pkg_dict = p.toolkit.get_action('package_show')(
+                None, {'id': id}
+            )
+            c.resource = p.toolkit.get_action('resource_show')(
+                None, {'id': resource_id}
+            )
+        except logic.NotFound:
+            base.abort(404, _('Resource not found'))
+        except logic.NotAuthorized:
+            base.abort(401, _('Unauthorized to edit this resource'))
+
+        try:
+            datapusher_status = p.toolkit.get_action('datapusher_status')(
+                None, {'resource_id': resource_id}
+            )
+        except logic.NotFound:
+            datapusher_status = {}
+
+        return base.render('package/resource_data.html',
+                            extra_vars={'status': datapusher_status})
+
 
 class DatapusherPlugin(p.SingletonPlugin):
     p.implements(p.IConfigurable, inherit=True)
@@ -24,6 +69,7 @@ class DatapusherPlugin(p.SingletonPlugin):
     p.implements(p.IResourceUrlChange)
     p.implements(p.IDomainObjectModification, inherit=True)
     p.implements(p.ITemplateHelpers)
+    p.implements(p.IRoutes, inherit=True)
 
     legacy_mode = False
     resource_show_action = None
@@ -39,6 +85,11 @@ def configure(self, config):
             raise Exception(
                 'Config option `ckan.datapusher.url` has to be set.')
 
+        self.datapusher_secret_key = config.get('ckan.datapusher.secret_key', '')
+        if not datapusher_url:
+            raise Exception(
+                'Config option `ckan.datapusher.secret_key` has to be set.')
+
     def notify(self, entity, operation=None):
         if isinstance(entity, model.Resource):
             if (operation == model.domain_object.DomainObjectOperation.new
@@ -63,6 +114,11 @@ def notify(self, entity, operation=None):
                         log.critical(e)
                         pass
 
+    def before_map(self, m):
+        m.connect('resource_data', '/dataset/{id}/resource_data/{resource_id}',
+                  controller='ckanext.datapusher.plugin:ResourceDataController',
+                  action='resource_data')
+        return m
 
     def get_actions(self):
         return {'datapusher_submit': action.datapusher_submit,
diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py
index 3bb5def4e73..f4a14a352c6 100644
--- a/ckanext/datastore/controller.py
+++ b/ckanext/datastore/controller.py
@@ -42,3 +42,4 @@ def dump(self, resource_id):
         for record in result['records']:
             wr.writerow([record[column] for column in header])
         return f.getvalue()
+

From dcf8761053e91f635b1febd6e25e97722272cd38 Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Fri, 20 Sep 2013 18:25:05 +0100
Subject: [PATCH 31/87] [1200] pep8 and test failure fixups

---
 ckan/logic/action/get.py           |  4 ++--
 ckanext/datapusher/logic/action.py | 10 +++++-----
 ckanext/datapusher/plugin.py       | 30 ++++++++++++++++--------------
 ckanext/datapusher/tests/test.py   | 20 +++++---------------
 ckanext/datastore/controller.py    |  1 -
 5 files changed, 28 insertions(+), 37 deletions(-)

diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py
index 7c9cdfddace..d02f6e1578b 100644
--- a/ckan/logic/action/get.py
+++ b/ckan/logic/action/get.py
@@ -1834,11 +1834,11 @@ def task_status_show(context, data_dict):
 
     context['task_status'] = task_status
 
+    _check_access('task_status_show', context, data_dict)
+
     if task_status is None:
         raise NotFound
 
-    _check_access('task_status_show', context, data_dict)
-
     task_status_dict = model_dictize.task_status_dictize(task_status, context)
     return task_status_dict
 
diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py
index 653dd44e0b5..66c56d22c82 100644
--- a/ckanext/datapusher/logic/action.py
+++ b/ckanext/datapusher/logic/action.py
@@ -115,10 +115,9 @@ def datapusher_submit(context, data_dict):
         p.toolkit.get_action('task_status_update')(context, task)
         raise p.toolkit.ValidationError(error)
 
-    value = json.dumps(
-            {'job_id': r.json()['job_id'],
-             'job_key': r.json()['job_key']}
-            )
+    value = json.dumps({'job_id': r.json()['job_id'],
+                        'job_key': r.json()['job_key']})
+
     task['value'] = value
     task['state'] = 'pending'
     task['last_updated'] = str(datetime.datetime.now()),
@@ -187,7 +186,8 @@ def datapusher_status(context, data_dict):
             r = requests.get(url, headers={'Content-Type': 'application/json',
                                            'Authorization': job_key})
             job_detail = r.json()
-        except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e:
+        except (requests.exceptions.ConnectionError,
+                requests.exceptions.HTTPError), e:
             job_detail = {'error': 'cannot connect to datapusher'}
 
     return {
diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py
index a545323a7dd..2138497ad57 100644
--- a/ckanext/datapusher/plugin.py
+++ b/ckanext/datapusher/plugin.py
@@ -19,6 +19,7 @@
 class DatastoreException(Exception):
     pass
 
+
 class ResourceDataController(base.BaseController):
 
     def resource_data(self, id, resource_id):
@@ -33,10 +34,10 @@ def resource_data(self, id, resource_id):
                 pass
 
             base.redirect(core_helpers.url_for(
-                 controller='ckanext.datapusher.plugin:ResourceDataController',
-                 action='resource_data',
-                 id=id,
-                 resource_id=resource_id)
+                controller='ckanext.datapusher.plugin:ResourceDataController',
+                action='resource_data',
+                id=id,
+                resource_id=resource_id)
             )
 
         try:
@@ -59,7 +60,7 @@ def resource_data(self, id, resource_id):
             datapusher_status = {}
 
         return base.render('package/resource_data.html',
-                            extra_vars={'status': datapusher_status})
+                           extra_vars={'status': datapusher_status})
 
 
 class DatapusherPlugin(p.SingletonPlugin):
@@ -77,15 +78,14 @@ class DatapusherPlugin(p.SingletonPlugin):
     def configure(self, config):
         self.config = config
 
-        datapusher_formats = config.get('ckan.datapusher.formats', '').lower().split()
-        self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS
+        datapusher_formats = config.get('ckan.datapusher.formats', '').lower()
+        self.datapusher_formats = datapusher_formats.split() or DEFAULT_FORMATS
 
         datapusher_url = config.get('ckan.datapusher.url')
         if not datapusher_url:
             raise Exception(
                 'Config option `ckan.datapusher.url` has to be set.')
 
-        self.datapusher_secret_key = config.get('ckan.datapusher.secret_key', '')
         if not datapusher_url:
             raise Exception(
                 'Config option `ckan.datapusher.secret_key` has to be set.')
@@ -97,7 +97,8 @@ def notify(self, entity, operation=None):
                 # if operation is None, resource URL has been changed, as
                 # the notify function in IResourceUrlChange only takes
                 # 1 parameter
-                context = {'model': model, 'ignore_auth': True, 'defer_commit': True}
+                context = {'model': model, 'ignore_auth': True,
+                           'defer_commit': True}
                 package = p.toolkit.get_action('package_show')(context, {
                     'id': entity.get_package_id()
                 })
@@ -109,15 +110,16 @@ def notify(self, entity, operation=None):
                             'resource_id': entity.id
                         })
                     except p.toolkit.ValidationError, e:
-                        # If datapusher is offline want to catch error instead of
-                        # raising otherwise resource save will fail with 500
+                        # If datapusher is offline want to catch error instead
+                        # of raising otherwise resource save will fail with 500
                         log.critical(e)
                         pass
 
     def before_map(self, m):
-        m.connect('resource_data', '/dataset/{id}/resource_data/{resource_id}',
-                  controller='ckanext.datapusher.plugin:ResourceDataController',
-                  action='resource_data')
+        m.connect(
+            'resource_data', '/dataset/{id}/resource_data/{resource_id}',
+            controller='ckanext.datapusher.plugin:ResourceDataController',
+            action='resource_data')
         return m
 
     def get_actions(self):
diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py
index f47f256cef6..9e8e88a939c 100644
--- a/ckanext/datapusher/tests/test.py
+++ b/ckanext/datapusher/tests/test.py
@@ -131,7 +131,7 @@ def test_send_datapusher_creates_task(self):
         task = p.toolkit.get_action('task_status_show')(context, {
             'entity_id': resource.id,
             'task_type': 'datapusher',
-            'key': 'job_id'
+            'key': 'datapusher'
         })
 
         assert task['state'] == 'pending', task
@@ -148,18 +148,8 @@ def test_datapusher_hook(self):
             'entity_id': resource.id,
             'entity_type': 'resource',
             'task_type': 'datapusher',
-            'key': 'job_id',
-            'value': 'my_id',
-            'last_updated': str(datetime.datetime.now()),
-            'state': 'pending'
-        })
-
-        p.toolkit.get_action('task_status_update')(context, {
-            'entity_id': resource.id,
-            'entity_type': 'resource',
-            'task_type': 'datapusher',
-            'key': 'job_key',
-            'value': 'my_key',
+            'key': 'datapusher',
+            'value': '{"job_id": "my_id", "job_key":"my_key"}',
             'last_updated': str(datetime.datetime.now()),
             'state': 'pending'
         })
@@ -181,12 +171,12 @@ def test_datapusher_hook(self):
 
         task = tests.call_action_api(
             self.app, 'task_status_show', entity_id=resource.id,
-            task_type='datapusher', key='job_id')
+            task_type='datapusher', key='datapusher')
 
         assert task['state'] == 'success', task
 
         task = tests.call_action_api(
             self.app, 'task_status_show', entity_id=resource.id,
-            task_type='datapusher', key='job_key')
+            task_type='datapusher', key='datapusher')
 
         assert task['state'] == 'success', task
diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py
index f4a14a352c6..3bb5def4e73 100644
--- a/ckanext/datastore/controller.py
+++ b/ckanext/datastore/controller.py
@@ -42,4 +42,3 @@ def dump(self, resource_id):
         for record in result['records']:
             wr.writerow([record[column] for column in header])
         return f.getvalue()
-

From e39f6362e6c682703da7eb65f32ff7e7375e35ef Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Fri, 20 Sep 2013 21:21:23 +0100
Subject: [PATCH 32/87] [1200] fix task_staus in context

---
 ckanext/datapusher/tests/test.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py
index 9e8e88a939c..2ec2df42d21 100644
--- a/ckanext/datapusher/tests/test.py
+++ b/ckanext/datapusher/tests/test.py
@@ -128,6 +128,8 @@ def test_send_datapusher_creates_task(self):
             'resource_id': resource.id
         })
 
+        context.pop('task_status', None)
+
         task = p.toolkit.get_action('task_status_show')(context, {
             'entity_id': resource.id,
             'task_type': 'datapusher',

From 3d5fe71e8b17067f032784be75b932a352163fc6 Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Mon, 7 Oct 2013 10:00:07 +0100
Subject: [PATCH 33/87] [1262] basic image upload functionality

---
 ckan/config/middleware.py                     |  6 ++
 ckan/controllers/group.py                     | 28 ++++++++-
 ckan/lib/dictization/model_dictize.py         | 21 +++++++
 ckan/lib/helpers.py                           |  5 ++
 ckan/lib/munge.py                             |  7 +++
 ckan/lib/uploader.py                          | 63 +++++++++++++++++++
 ckan/logic/schema.py                          |  1 +
 ckan/templates/group/snippets/group_form.html | 11 +++-
 ckan/templates/group/snippets/group_item.html |  2 +-
 .../snippets/organization_form.html           | 10 ++-
 .../snippets/organization_item.html           |  2 +-
 ckan/templates/snippets/group.html            |  2 +-
 ckan/templates/snippets/group_item.html       |  2 +-
 ckan/templates/snippets/organization.html     |  2 +-
 .../templates/snippets/organization_item.html |  2 +-
 15 files changed, 155 insertions(+), 9 deletions(-)
 create mode 100644 ckan/lib/uploader.py

diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py
index 4df0aa84cb0..63f5568cd5f 100644
--- a/ckan/config/middleware.py
+++ b/ckan/config/middleware.py
@@ -147,6 +147,12 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf):
                 cache_max_age=static_max_age)
         static_parsers = [static_app, app]
 
+        storage_directory = config.get('ckan.storage_path')
+        if storage_directory:
+            storage_app = StaticURLParser(storage_directory,
+                cache_max_age=static_max_age)
+            static_parsers.insert(0, storage_app)
+
         # Configurable extra static file paths
         extra_static_parsers = []
         for public_path in config.get('extra_public_paths', '').split(','):
diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py
index ccef46bfe78..a11dd03cc40 100644
--- a/ckan/controllers/group.py
+++ b/ckan/controllers/group.py
@@ -1,14 +1,18 @@
 import re
+import os
 import logging
 import genshi
+import cgi
 import datetime
 from urllib import urlencode
 
 from pylons.i18n import get_lang
+from pylons import config
 
 import ckan.lib.base as base
 import ckan.lib.helpers as h
 import ckan.lib.maintain as maintain
+import ckan.lib.uploader as uploader
 import ckan.lib.navl.dictization_functions as dict_fns
 import ckan.logic as logic
 import ckan.lib.search as search
@@ -421,6 +425,9 @@ def new(self, data=None, errors=None, error_summary=None):
             return self._save_new(context, group_type)
 
         data = data or {}
+        if not data.get('image_url', '').startswith('http'):
+            data.pop('image_url', None)
+
         errors = errors or {}
         error_summary = error_summary or {}
         vars = {'data': data, 'errors': errors,
@@ -445,10 +452,17 @@ def edit(self, id, data=None, errors=None, error_summary=None):
             return self._save_edit(id, context)
 
         try:
+            ## only needs removing when form repopulates
+            if data and not data.get('image_url', '').startswith('http'):
+                data.pop('image_url', None)
             old_data = self._action('group_show')(context, data_dict)
             c.grouptitle = old_data.get('title')
             c.groupname = old_data.get('name')
             data = data or old_data
+            image_url = data.get('image_url')
+            if image_url and not image_url.startswith('http'):
+                data['upload_file'] =  image_url
+                data['image_url'] = ''
         except NotFound:
             abort(404, _('Group not found'))
         except NotAuthorized:
@@ -456,6 +470,7 @@ def edit(self, id, data=None, errors=None, error_summary=None):
 
         group = context.get("group")
         c.group = group
+        context.pop('for_edit')
         c.group_dict = self._action('group_show')(context, data_dict)
 
         try:
@@ -483,12 +498,17 @@ def _get_group_type(self, id):
 
     def _save_new(self, context, group_type=None):
         try:
+            upload = uploader.Upload('group')
+            upload.register_request(request, 'image_url',
+                                    'image_upload', 'clear_upload')
+
             data_dict = clean_dict(dict_fns.unflatten(
                 tuplize_dict(parse_params(request.params))))
             data_dict['type'] = group_type or 'group'
             context['message'] = data_dict.get('log_message', '')
             data_dict['users'] = [{'name': c.user, 'capacity': 'admin'}]
             group = self._action('group_create')(context, data_dict)
+            upload.upload()
 
             # Redirect to the appropriate _read route for the type of group
             h.redirect_to(group['type'] + '_read', id=group['name'])
@@ -514,13 +534,19 @@ def _force_reindex(self, grp):
 
     def _save_edit(self, id, context):
         try:
+            old_group = self._action('group_show')(context, {"id": id})
+            old_image_url = old_group.get('image_url')
+            upload = uploader.Upload('group', old_image_url)
+            upload.register_request(request, 'image_url',
+                                    'image_upload', 'clear_upload')
+
             data_dict = clean_dict(dict_fns.unflatten(
                 tuplize_dict(parse_params(request.params))))
             context['message'] = data_dict.get('log_message', '')
             data_dict['id'] = id
             context['allow_partial_update'] = True
             group = self._action('group_update')(context, data_dict)
-
+            upload.upload()
             if id != group['name']:
                 self._force_reindex(group)
 
diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py
index b3a83235cd6..98756abc2ec 100644
--- a/ckan/lib/dictization/model_dictize.py
+++ b/ckan/lib/dictization/model_dictize.py
@@ -10,6 +10,7 @@
 import ckan.lib.dictization as d
 import ckan.new_authz as new_authz
 import ckan.lib.search as search
+import ckan.lib.munge as munge
 
 ## package save
 
@@ -41,6 +42,16 @@ def group_list_dictize(obj_list, context,
 
         group_dict['display_name'] = obj.display_name
 
+        image_url = group_dict.get('image_url')
+        group_dict['image_display_url'] = image_url
+        if image_url and not image_url.startswith('http'):
+            #munge here should not have an effect only doing it incase
+            #of potential vulnerability of dodgy api input
+            image_url = munge.munge_filename(image_url)
+            group_dict['image_display_url'] = h.url_for_static(
+                'uploads/group/%s' % group_dict.get('image_url')
+            )
+
         if obj.is_organization:
             group_dict['packages'] = query.facets['owner_org'].get(obj.id, 0)
         else:
@@ -354,6 +365,16 @@ def group_dictize(group, context):
         for item in plugins.PluginImplementations(plugin):
             result_dict = item.before_view(result_dict)
 
+    if not context.get('for_edit'):
+        image_url = result_dict.get('image_url')
+        result_dict['image_display_url'] = image_url
+        if image_url and not image_url.startswith('http'):
+            #munge here should not have an effect only doing it incase
+            #of potential vulnerability of dodgy api input
+            image_url = munge.munge_filename(image_url)
+            result_dict['image_display_url'] = h.url_for_static(
+                'uploads/group/%s' % result_dict.get('image_url')
+            )
     return result_dict
 
 def tag_list_dictize(tag_list, context):
diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
index c70d74d7dbc..fceab6f1b6a 100644
--- a/ckan/lib/helpers.py
+++ b/ckan/lib/helpers.py
@@ -1663,6 +1663,10 @@ def new_activities():
     action = logic.get_action('dashboard_new_activities_count')
     return action({}, {})
 
+def uploads_enabled():
+    if config.get('ckan.storage_path'):
+        return True
+    return False
 
 # these are the functions that will end up in `h` template helpers
 __allowed_functions__ = [
@@ -1761,4 +1765,5 @@ def new_activities():
     'radio',
     'submit',
     'asbool',
+    'uploads_enabled',
 ]
diff --git a/ckan/lib/munge.py b/ckan/lib/munge.py
index c943e1b113f..a47c8c02258 100644
--- a/ckan/lib/munge.py
+++ b/ckan/lib/munge.py
@@ -105,6 +105,13 @@ def munge_tag(tag):
     tag = _munge_to_length(tag, model.MIN_TAG_LENGTH, model.MAX_TAG_LENGTH)
     return tag
 
+def munge_filename(filename):
+    filename = substitute_ascii_equivalents(filename)
+    filename = filename.lower().strip()
+    filename = re.sub(r'[^a-zA-Z0-9. ]', '', filename).replace(' ', '-')
+    filename = _munge_to_length(filename, 3, 100)
+    return filename
+
 def _munge_to_length(string, min_length, max_length):
     '''Pad/truncates a string'''
     if len(string) < min_length:
diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py
new file mode 100644
index 00000000000..5fc144e3aef
--- /dev/null
+++ b/ckan/lib/uploader.py
@@ -0,0 +1,63 @@
+import os
+import cgi
+import pylons
+import datetime
+import ckan.lib.munge as munge
+
+class Upload(object):
+    def __init__(self, object_type, old_filename=None):
+        path = pylons.config.get('ckan.storage_path', '/tmp')
+        self.storage_path = os.path.join(path, 'uploads', object_type)
+        try:
+            os.makedirs(self.storage_path)
+        except OSError, e:
+            pass
+        self.object_type = object_type
+        self.old_filename = old_filename
+        if old_filename:
+            self.old_filepath = os.path.join(self.storage_path, old_filename)
+        self.filename = None
+        self.filepath = None
+
+    def register_request(self, request, url_field, file_field, clear_field):
+
+        self.request = request
+        self.url = request.POST.get(url_field)
+        self.clear = request.POST.get(clear_field)
+        self.upload_field_storage = request.POST.pop(file_field, None)
+
+        if isinstance(self.upload_field_storage, cgi.FieldStorage):
+            self.filename = self.upload_field_storage.filename
+            self.filename = str(datetime.datetime.utcnow()) + self.filename
+            self.filename = munge.munge_filename(self.filename)
+            self.filepath = os.path.join(self.storage_path, self.filename)
+            request.POST[url_field] = self.filename
+            self.upload_file = self.upload_field_storage.file
+            self.tmp_filepath = self.filepath + '~'
+        ### keep the file if there has been no change
+        if self.old_filename and not self.url and not self.clear:
+            request.POST[url_field] = self.old_filename
+
+
+    def upload(self):
+
+        if self.filename:
+            output_file = open(self.tmp_filepath, 'wb')
+            self.upload_file.seek(0)
+            while True:
+                data = self.upload_file.read(2 ** 20) #mb chuncks
+                if not data:
+                    break
+                output_file.write(data)
+            output_file.close()
+            os.rename(self.tmp_filepath, self.filepath)
+            self.clear = True
+        elif self.url:
+            self.clear = True
+
+        if (self.clear and self.old_filename
+            and not self.old_filename.startswith('http')):
+            try:
+                os.remove(self.old_filepath)
+            except OSError, e:
+                pass
diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py
index f8fe8df7fbd..90465b7f7ec 100644
--- a/ckan/logic/schema.py
+++ b/ckan/logic/schema.py
@@ -262,6 +262,7 @@ def default_group_schema():
         'title': [ignore_missing, unicode],
         'description': [ignore_missing, unicode],
         'image_url': [ignore_missing, unicode],
+        'image_display_url': [ignore_missing, unicode],
         'type': [ignore_missing, unicode],
         'state': [ignore_not_group_admin, ignore_missing],
         'created': [ignore],
diff --git a/ckan/templates/group/snippets/group_form.html b/ckan/templates/group/snippets/group_form.html
index 6d4d3216b26..1b9cac179c7 100644
--- a/ckan/templates/group/snippets/group_form.html
+++ b/ckan/templates/group/snippets/group_form.html
@@ -1,6 +1,6 @@
 {% import 'macros/form.html' as form %}
 
-<form class="dataset-form form-horizontal" method="post" data-module="basic-form">
+<form class="dataset-form form-horizontal" method="post" data-module="basic-form" enctype="multipart/form-data">
   {% block error_summary %}
     {{ form.errors(error_summary) }}
   {% endblock %}
@@ -21,6 +21,15 @@
 
     {{ form.input('image_url', label=_('Image URL'), id='field-image-url', type='url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }}
 
+
+    {% if h.uploads_enabled() %}
+    {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }}
+      {% set upload_file = data.get('upload_file', '') %}
+      {% if upload_file %}
+        {{ form.checkbox('clear_upload', label=_('Clear Upload') + ' ' + upload_file, id='field-clear-upload', value='true', error='', classes=['control-full']) }}
+      {% endif %}
+    {% endif %}
+
   {% endblock %}
 
   {% block custom_fields %}
diff --git a/ckan/templates/group/snippets/group_item.html b/ckan/templates/group/snippets/group_item.html
index 9e32ce86e6a..34688b57c96 100644
--- a/ckan/templates/group/snippets/group_item.html
+++ b/ckan/templates/group/snippets/group_item.html
@@ -14,7 +14,7 @@
 {% set url = h.url_for(group.type ~ '_read', action='read', id=group.name) %}
 <li class="media-item">
   {% block image %}
-    <img src="{{ group.image_url or '/base/images/placeholder-group.png' }}" alt="{{ group.name }}" class="media-image">
+    <img src="{{ group.image_display_url or '/base/images/placeholder-group.png' }}" alt="{{ group.name }}" class="media-image">
   {% endblock %}
   {% block title %}
     <h3 class="media-heading">{{ group.display_name }}</h3>
diff --git a/ckan/templates/organization/snippets/organization_form.html b/ckan/templates/organization/snippets/organization_form.html
index 7b05e7d1542..7aee9ded32f 100644
--- a/ckan/templates/organization/snippets/organization_form.html
+++ b/ckan/templates/organization/snippets/organization_form.html
@@ -1,6 +1,6 @@
 {% import 'macros/form.html' as form %}
 
-<form class="dataset-form form-horizontal" method="post" data-module="basic-form">
+<form class="dataset-form form-horizontal" method="post" data-module="basic-form" enctype="multipart/form-data">
   {% block error_summary %}
     {{ form.errors(error_summary) }}
   {% endblock %}
@@ -21,6 +21,14 @@
 
     {{ form.input('image_url', label=_('Image URL'), id='field-image-url', type='url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }}
 
+    {% if h.uploads_enabled() %}
+    {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }}
+      {% set upload_file = data.get('upload_file', '') %}
+      {% if upload_file %}
+        {{ form.checkbox('clear_upload', label=_('Clear Upload') + ' ' + upload_file, id='field-clear-upload', value='true', error='', classes=['control-full']) }}
+      {% endif %}
+    {% endif %}
+
   {% endblock %}
 
   {% block custom_fields %}
diff --git a/ckan/templates/organization/snippets/organization_item.html b/ckan/templates/organization/snippets/organization_item.html
index 818d5cc338a..8118fa9a1c6 100644
--- a/ckan/templates/organization/snippets/organization_item.html
+++ b/ckan/templates/organization/snippets/organization_item.html
@@ -14,7 +14,7 @@
 {% set url = h.url_for(organization.type ~ '_read', action='read', id=organization.name) %}
 <li class="media-item">
   {% block image %}
-    <img src="{{ organization.image_url or '/base/images/placeholder-organization.png' }}" alt="{{ organization.name }}" class="media-image">
+    <img src="{{ organization.image_display_url or '/base/images/placeholder-organization.png' }}" alt="{{ organization.name }}" class="media-image">
   {% endblock %}
   {% block title %}
     <h3 class="media-heading">{{ organization.display_name }}</h3>
diff --git a/ckan/templates/snippets/group.html b/ckan/templates/snippets/group.html
index db549fda75a..db758d9bc96 100644
--- a/ckan/templates/snippets/group.html
+++ b/ckan/templates/snippets/group.html
@@ -14,7 +14,7 @@
   <section class="module module-narrow module-shallow group">
     <div class="module-content media media-vertical">
       <a class="media-image" href="{{ url }}">
-        <img src="{{ group.image_url or h.url_for_static('/base/images/placeholder-group.png') }}" width="200" height="125" alt="{{ group.name }}" />
+        <img src="{{ group.image_display_url or h.url_for_static('/base/images/placeholder-group.png') }}" width="200" height="125" alt="{{ group.name }}" />
       </a>
       <div class="media-content">
         <h3 class="media-heading"><a href={{ url }}>{{ group.title or group.name }}</a></h3>
diff --git a/ckan/templates/snippets/group_item.html b/ckan/templates/snippets/group_item.html
index 74c68fbca28..60f43f63b79 100644
--- a/ckan/templates/snippets/group_item.html
+++ b/ckan/templates/snippets/group_item.html
@@ -5,7 +5,7 @@
     {% set truncate_title = truncate_title or 0 %}
     {% set title = group.title or group.name %}
     <a class="module-image" href="{{ url }}">
-      <img src="{{ group.image_url or h.url_for_static('/base/images/placeholder-group.png') }}" alt="{{ group.name }}" />
+      <img src="{{ group.image_display_url or h.url_for_static('/base/images/placeholder-group.png') }}" alt="{{ group.name }}" />
     </a>
     <h3 class="media-heading"><a href="{{ url }}">{{ group.title or group.name }}</a></h3>
     {% if group.description %}
diff --git a/ckan/templates/snippets/organization.html b/ckan/templates/snippets/organization.html
index ef08a9ed440..e0f5417024a 100644
--- a/ckan/templates/snippets/organization.html
+++ b/ckan/templates/snippets/organization.html
@@ -23,7 +23,7 @@ <h2 class="module-heading"><i class="icon-building"></i> {{ _('Organization') }}
       {% block image %}
         <div class="image">
           <a href="{{ url }}">
-            <img src="{{ organization.image_url or h.url_for_static('/base/images/placeholder-organization.png') }}" width="200" alt="{{ organization.name }}" />
+            <img src="{{ organization.image_display_url or h.url_for_static('/base/images/placeholder-organization.png') }}" width="200" alt="{{ organization.name }}" />
           </a>
         </div>
       {% endblock %}
diff --git a/ckan/templates/snippets/organization_item.html b/ckan/templates/snippets/organization_item.html
index 0213723defa..06579756f32 100644
--- a/ckan/templates/snippets/organization_item.html
+++ b/ckan/templates/snippets/organization_item.html
@@ -3,7 +3,7 @@
     {% set url=h.url_for(controller='organization', action='read', id=organization.name) %}
     {% set truncate=truncate or 0 %}
     <a class="module-image" href="{{ url }}">
-      <img src="{{ organization.image_url or h.url_for_static('/base/images/placeholder-organization.png') }}" alt="{{ organization.name }}" />
+      <img src="{{ organization.image_display_url or h.url_for_static('/base/images/placeholder-organization.png') }}" alt="{{ organization.name }}" />
     </a>
     <h3 class="media-heading"><a href={{ url }}>{{ organization.title or organization.name }}</a></h3>
     {% if organization.description %}

From 66105d360e010114751ff0280b07196e6f73445a Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Tue, 8 Oct 2013 03:02:12 +0100
Subject: [PATCH 34/87] [#1262] fix how url field is handled

---
 ckan/config/middleware.py                     |  9 ++++++++-
 ckan/controllers/group.py                     |  7 -------
 ckan/lib/dictization/model_dictize.py         | 20 +++++++++----------
 ckan/lib/uploader.py                          | 16 +++++++--------
 ckan/templates/group/snippets/group_form.html |  7 +++----
 ckan/templates/group/snippets/info.html       |  2 +-
 6 files changed, 29 insertions(+), 32 deletions(-)

diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py
index 63f5568cd5f..7699596138d 100644
--- a/ckan/config/middleware.py
+++ b/ckan/config/middleware.py
@@ -4,6 +4,7 @@
 import logging
 import json
 import hashlib
+import os
 
 import sqlalchemy as sa
 from beaker.middleware import CacheMiddleware, SessionMiddleware
@@ -149,7 +150,13 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf):
 
         storage_directory = config.get('ckan.storage_path')
         if storage_directory:
-            storage_app = StaticURLParser(storage_directory,
+            path = os.path.join(storage_directory, 'storage')
+            try:
+                os.makedirs(path)
+            except OSError, e:
+                pass
+
+            storage_app = StaticURLParser(path,
                 cache_max_age=static_max_age)
             static_parsers.insert(0, storage_app)
 
diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py
index a11dd03cc40..ed12edcb082 100644
--- a/ckan/controllers/group.py
+++ b/ckan/controllers/group.py
@@ -452,17 +452,10 @@ def edit(self, id, data=None, errors=None, error_summary=None):
             return self._save_edit(id, context)
 
         try:
-            ## only needs removing when form repopulates
-            if data and not data.get('image_url', '').startswith('http'):
-                data.pop('image_url', None)
             old_data = self._action('group_show')(context, data_dict)
             c.grouptitle = old_data.get('title')
             c.groupname = old_data.get('name')
             data = data or old_data
-            image_url = data.get('image_url')
-            if image_url and not image_url.startswith('http'):
-                data['upload_file'] =  image_url
-                data['image_url'] = ''
         except NotFound:
             abort(404, _('Group not found'))
         except NotAuthorized:
diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py
index 98756abc2ec..019e648140b 100644
--- a/ckan/lib/dictization/model_dictize.py
+++ b/ckan/lib/dictization/model_dictize.py
@@ -365,16 +365,16 @@ def group_dictize(group, context):
         for item in plugins.PluginImplementations(plugin):
             result_dict = item.before_view(result_dict)
 
-    if not context.get('for_edit'):
-        image_url = result_dict.get('image_url')
-        result_dict['image_display_url'] = image_url
-        if image_url and not image_url.startswith('http'):
-            #munge here should not have an effect only doing it incase
-            #of potential vulnerability of dodgy api input
-            image_url = munge.munge_filename(image_url)
-            result_dict['image_display_url'] = h.url_for_static(
-                'uploads/group/%s' % result_dict.get('image_url')
-            )
+    image_url = result_dict.get('image_url')
+    result_dict['image_display_url'] = image_url
+    if image_url and not image_url.startswith('http'):
+        #munge here should not have an effect only doing it incase
+        #of potential vulnerability of dodgy api input
+        image_url = munge.munge_filename(image_url)
+        result_dict['image_display_url'] = h.url_for_static(
+            'uploads/group/%s' % result_dict.get('image_url'),
+            qualified = True
+        )
     return result_dict
 
 def tag_list_dictize(tag_list, context):
diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py
index 5fc144e3aef..bcfdd7e9ecc 100644
--- a/ckan/lib/uploader.py
+++ b/ckan/lib/uploader.py
@@ -7,7 +7,7 @@
 class Upload(object):
     def __init__(self, object_type, old_filename=None):
         path = pylons.config.get('ckan.storage_path', '/tmp')
-        self.storage_path = os.path.join(path, 'uploads', object_type)
+        self.storage_path = os.path.join(path, 'storage', 'uploads', object_type)
         try:
             os.makedirs(self.storage_path)
         except OSError, e:
@@ -20,9 +20,7 @@ def __init__(self, object_type, old_filename=None):
         self.filepath = None
 
     def register_request(self, request, url_field, file_field, clear_field):
-
-        self.request = request
-        self.url = request.POST.get(url_field)
+        self.url = request.POST.get(url_field, '')
         self.clear = request.POST.get(clear_field)
         self.upload_field_storage = request.POST.pop(file_field, None)
 
@@ -35,12 +33,14 @@ def register_request(self, request, url_field, file_field, clear_field):
             self.upload_file = self.upload_field_storage.file
             self.tmp_filepath = self.filepath + '~'
         ### keep the file if there has been no change
-        if self.old_filename and not self.url and not self.clear:
-            request.POST[url_field] = self.old_filename
+        elif self.old_filename and not self.old_filename.startswith('http'):
+            if not self.clear:
+                request.POST[url_field] = self.old_filename
+            if self.clear and self.url == self.old_filename:
+                request.POST[url_field] = ''
 
 
     def upload(self):
-
         if self.filename:
             output_file = open(self.tmp_filepath, 'wb')
             self.upload_file.seek(0)
@@ -52,8 +52,6 @@ def upload(self):
             output_file.close()
             os.rename(self.tmp_filepath, self.filepath)
             self.clear = True
-        elif self.url:
-            self.clear = True
 
         if (self.clear and self.old_filename
             and not self.old_filename.startswith('http')):
diff --git a/ckan/templates/group/snippets/group_form.html b/ckan/templates/group/snippets/group_form.html
index 1b9cac179c7..322f9ebff8a 100644
--- a/ckan/templates/group/snippets/group_form.html
+++ b/ckan/templates/group/snippets/group_form.html
@@ -19,14 +19,13 @@
 
     {{ form.markdown('description', label=_('Description'), id='field-description', placeholder=_('A little information about my group...'), value=data.description, error=errors.description) }}
 
-    {{ form.input('image_url', label=_('Image URL'), id='field-image-url', type='url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }}
+    {{ form.input('image_url', label=_('Image URL'), id='field-image-url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }}
 
 
     {% if h.uploads_enabled() %}
     {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }}
-      {% set upload_file = data.get('upload_file', '') %}
-      {% if upload_file %}
-        {{ form.checkbox('clear_upload', label=_('Clear Upload') + ' ' + upload_file, id='field-clear-upload', value='true', error='', classes=['control-full']) }}
+      {% if data.image_url and not data.image_url.startswith('http') %}
+        {{ form.checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }}
       {% endif %}
     {% endif %}
 
diff --git a/ckan/templates/group/snippets/info.html b/ckan/templates/group/snippets/info.html
index b7c749ab147..dc86036bb4c 100644
--- a/ckan/templates/group/snippets/info.html
+++ b/ckan/templates/group/snippets/info.html
@@ -2,7 +2,7 @@
   <section class="module-content">
     <div class="image">
       <a href="{{ group.url }}">
-        <img src="{{ group.image_url or h.url_for_static('/base/images/placeholder-group.png') }}" width="190" height="118" alt="{{ group.name }}" />
+        <img src="{{ group.image_display_url or h.url_for_static('/base/images/placeholder-group.png') }}" width="190" height="118" alt="{{ group.name }}" />
       </a>
     </div>
     <h1 class="heading">{{ group.display_name }}</h1>

From 1b62423d9d013dbc4481bbddd5a1037b49e52529 Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Wed, 9 Oct 2013 03:12:00 +0100
Subject: [PATCH 35/87] [1262] use logic instead of requests

---
 ckan/controllers/group.py                          | 11 -----------
 ckan/lib/uploader.py                               | 14 +++++++-------
 ckan/logic/action/create.py                        |  7 ++++++-
 ckan/logic/action/update.py                        |  7 +++++++
 .../organization/snippets/organization_form.html   |  7 +++----
 5 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py
index ed12edcb082..adbd3d7cacc 100644
--- a/ckan/controllers/group.py
+++ b/ckan/controllers/group.py
@@ -12,7 +12,6 @@
 import ckan.lib.base as base
 import ckan.lib.helpers as h
 import ckan.lib.maintain as maintain
-import ckan.lib.uploader as uploader
 import ckan.lib.navl.dictization_functions as dict_fns
 import ckan.logic as logic
 import ckan.lib.search as search
@@ -491,17 +490,12 @@ def _get_group_type(self, id):
 
     def _save_new(self, context, group_type=None):
         try:
-            upload = uploader.Upload('group')
-            upload.register_request(request, 'image_url',
-                                    'image_upload', 'clear_upload')
-
             data_dict = clean_dict(dict_fns.unflatten(
                 tuplize_dict(parse_params(request.params))))
             data_dict['type'] = group_type or 'group'
             context['message'] = data_dict.get('log_message', '')
             data_dict['users'] = [{'name': c.user, 'capacity': 'admin'}]
             group = self._action('group_create')(context, data_dict)
-            upload.upload()
 
             # Redirect to the appropriate _read route for the type of group
             h.redirect_to(group['type'] + '_read', id=group['name'])
@@ -529,17 +523,12 @@ def _save_edit(self, id, context):
         try:
             old_group = self._action('group_show')(context, {"id": id})
             old_image_url = old_group.get('image_url')
-            upload = uploader.Upload('group', old_image_url)
-            upload.register_request(request, 'image_url',
-                                    'image_upload', 'clear_upload')
-
             data_dict = clean_dict(dict_fns.unflatten(
                 tuplize_dict(parse_params(request.params))))
             context['message'] = data_dict.get('log_message', '')
             data_dict['id'] = id
             context['allow_partial_update'] = True
             group = self._action('group_update')(context, data_dict)
-            upload.upload()
             if id != group['name']:
                 self._force_reindex(group)
 
diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py
index bcfdd7e9ecc..9b7cea2922e 100644
--- a/ckan/lib/uploader.py
+++ b/ckan/lib/uploader.py
@@ -19,25 +19,25 @@ def __init__(self, object_type, old_filename=None):
         self.filename = None
         self.filepath = None
 
-    def register_request(self, request, url_field, file_field, clear_field):
-        self.url = request.POST.get(url_field, '')
-        self.clear = request.POST.get(clear_field)
-        self.upload_field_storage = request.POST.pop(file_field, None)
+    def update_data_dict(self, data_dict, url_field, file_field, clear_field):
+        self.url = data_dict.get(url_field, '')
+        self.clear = data_dict.pop(clear_field, None)
+        self.upload_field_storage =  data_dict.pop(file_field, None)
 
         if isinstance(self.upload_field_storage, cgi.FieldStorage):
             self.filename = self.upload_field_storage.filename
             self.filename = str(datetime.datetime.utcnow()) + self.filename
             self.filename = munge.munge_filename(self.filename)
             self.filepath = os.path.join(self.storage_path, self.filename)
-            request.POST[url_field] = self.filename
+            data_dict[url_field] = self.filename
             self.upload_file = self.upload_field_storage.file
             self.tmp_filepath = self.filepath + '~'
         ### keep the file if there has been no change
         elif self.old_filename and not self.old_filename.startswith('http'):
             if not self.clear:
-                request.POST[url_field] = self.old_filename
+                data_dict[url_field] = self.old_filename
             if self.clear and self.url == self.old_filename:
-                request.POST[url_field] = ''
+                data_dict[url_field] = ''
 
 
     def upload(self):
diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py
index baac5a2154c..702a6028d99 100644
--- a/ckan/logic/action/create.py
+++ b/ckan/logic/action/create.py
@@ -15,6 +15,7 @@
 import ckan.lib.dictization.model_dictize as model_dictize
 import ckan.lib.dictization.model_save as model_save
 import ckan.lib.navl.dictization_functions
+import ckan.lib.uploader as uploader
 
 from ckan.common import _
 
@@ -446,6 +447,7 @@ def member_create(context, data_dict=None):
     if not obj:
         raise NotFound('%s was not found.' % obj_type.title())
 
+
     # User must be able to update the group to add a member to it
     _check_access('group_update', context, data_dict)
 
@@ -475,7 +477,9 @@ def _group_or_org_create(context, data_dict, is_org=False):
     parent = context.get('parent', None)
     data_dict['is_organization'] = is_org
 
-
+    upload = uploader.Upload('group')
+    upload.update_data_dict(data_dict, 'image_url',
+                           'image_upload', 'clear_upload')
     # get the schema
     group_plugin = lib_plugins.lookup_group_plugin(
             group_type=data_dict.get('type'))
@@ -561,6 +565,7 @@ def _group_or_org_create(context, data_dict, is_org=False):
     logic.get_action('activity_create')(activity_create_context,
             activity_dict)
 
+    upload.upload()
     if not context.get('defer_commit'):
         model.repo.commit()
     context["group"] = group
diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py
index 3db2d0d19cf..f8aee9f62fb 100644
--- a/ckan/logic/action/update.py
+++ b/ckan/logic/action/update.py
@@ -19,6 +19,7 @@
 import ckan.lib.plugins as lib_plugins
 import ckan.lib.email_notifications as email_notifications
 import ckan.lib.search as search
+import ckan.lib.uploader as uploader
 
 from ckan.common import _, request
 
@@ -424,6 +425,10 @@ def _group_or_org_update(context, data_dict, is_org=False):
     except AttributeError:
         schema = group_plugin.form_to_db_schema()
 
+    upload = uploader.Upload('group', group.image_url)
+    upload.update_data_dict(data_dict, 'image_url',
+                           'image_upload', 'clear_upload')
+
     if is_org:
         _check_access('organization_update', context, data_dict)
     else:
@@ -528,9 +533,11 @@ def _group_or_org_update(context, data_dict, is_org=False):
         # TODO: Also create an activity detail recording what exactly changed
         # in the group.
 
+    upload.upload()
     if not context.get('defer_commit'):
         model.repo.commit()
 
+
     return model_dictize.group_dictize(group, context)
 
 def group_update(context, data_dict):
diff --git a/ckan/templates/organization/snippets/organization_form.html b/ckan/templates/organization/snippets/organization_form.html
index 7aee9ded32f..0a2134e89f7 100644
--- a/ckan/templates/organization/snippets/organization_form.html
+++ b/ckan/templates/organization/snippets/organization_form.html
@@ -19,13 +19,12 @@
 
     {{ form.markdown('description', label=_('Description'), id='field-description', placeholder=_('A little information about my organization...'), value=data.description, error=errors.description) }}
 
-    {{ form.input('image_url', label=_('Image URL'), id='field-image-url', type='url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }}
+    {{ form.input('image_url', label=_('Image URL'), id='field-image-url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }}
 
     {% if h.uploads_enabled() %}
     {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }}
-      {% set upload_file = data.get('upload_file', '') %}
-      {% if upload_file %}
-        {{ form.checkbox('clear_upload', label=_('Clear Upload') + ' ' + upload_file, id='field-clear-upload', value='true', error='', classes=['control-full']) }}
+      {% if data.image_url and not data.image_url.startswith('http') %}
+        {{ form.checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }}
       {% endif %}
     {% endif %}
 

From 8f8d2ab1a502100d7da4358ea2947fbf9a5c4adb Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Wed, 9 Oct 2013 11:28:26 +0100
Subject: [PATCH 36/87] 1200 make sure all users can submit job

---
 ckanext/datapusher/logic/action.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py
index 66c56d22c82..cfbd4620d09 100644
--- a/ckanext/datapusher/logic/action.py
+++ b/ckanext/datapusher/logic/action.py
@@ -71,6 +71,7 @@ def datapusher_submit(context, data_dict):
     except logic.NotFound:
         pass
 
+    context['ignore_auth'] = True
     result = p.toolkit.get_action('task_status_update')(context, task)
     task_id = result['id']
 

From c4d673327c6b83e01ead0091246cecb83875160a Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Thu, 10 Oct 2013 20:40:49 +0100
Subject: [PATCH 37/87] [#1262] fix test failures

---
 ckan/lib/uploader.py                      | 12 +++++++-----
 ckan/tests/lib/test_dictization.py        |  2 ++
 ckan/tests/lib/test_dictization_schema.py |  1 +
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py
index 9b7cea2922e..eb0f28eec54 100644
--- a/ckan/lib/uploader.py
+++ b/ckan/lib/uploader.py
@@ -4,10 +4,12 @@
 import datetime
 import ckan.lib.munge as munge
 
+
 class Upload(object):
     def __init__(self, object_type, old_filename=None):
         path = pylons.config.get('ckan.storage_path', '/tmp')
-        self.storage_path = os.path.join(path, 'storage', 'uploads', object_type)
+        self.storage_path = os.path.join(path, 'storage',
+                                         'uploads', object_type)
         try:
             os.makedirs(self.storage_path)
         except OSError, e:
@@ -22,7 +24,7 @@ def __init__(self, object_type, old_filename=None):
     def update_data_dict(self, data_dict, url_field, file_field, clear_field):
         self.url = data_dict.get(url_field, '')
         self.clear = data_dict.pop(clear_field, None)
-        self.upload_field_storage =  data_dict.pop(file_field, None)
+        self.upload_field_storage = data_dict.pop(file_field, None)
 
         if isinstance(self.upload_field_storage, cgi.FieldStorage):
             self.filename = self.upload_field_storage.filename
@@ -39,13 +41,13 @@ def update_data_dict(self, data_dict, url_field, file_field, clear_field):
             if self.clear and self.url == self.old_filename:
                 data_dict[url_field] = ''
 
-
     def upload(self):
         if self.filename:
             output_file = open(self.tmp_filepath, 'wb')
             self.upload_file.seek(0)
             while True:
-                data = self.upload_file.read(2 ** 20) #mb chuncks
+                # mb chuncks
+                data = self.upload_file.read(2 ** 20)
                 if not data:
                     break
                 output_file.write(data)
@@ -54,7 +56,7 @@ def upload(self):
             self.clear = True
 
         if (self.clear and self.old_filename
-            and not self.old_filename.startswith('http')):
+                and not self.old_filename.startswith('http')):
             try:
                 os.remove(self.old_filepath)
             except OSError, e:
diff --git a/ckan/tests/lib/test_dictization.py b/ckan/tests/lib/test_dictization.py
index 57f29963910..dcf955d3cc5 100644
--- a/ckan/tests/lib/test_dictization.py
+++ b/ckan/tests/lib/test_dictization.py
@@ -923,6 +923,7 @@ def test_16_group_dictized(self):
                                'capacity' : 'public',
                                'display_name': u'simple',
                                'image_url': u'',
+                               'image_display_url': u'',
                                'name': u'simple',
                                'packages': 0,
                                'state': u'active',
@@ -944,6 +945,7 @@ def test_16_group_dictized(self):
                     'name': u'help',
                     'display_name': u'help',
                     'image_url': u'',
+                    'image_display_url': u'',
                     'package_count': 2,
                     'is_organization': False,
                     'packages': [{'author': None,
diff --git a/ckan/tests/lib/test_dictization_schema.py b/ckan/tests/lib/test_dictization_schema.py
index 5a94d2e54e8..9507be2806f 100644
--- a/ckan/tests/lib/test_dictization_schema.py
+++ b/ckan/tests/lib/test_dictization_schema.py
@@ -160,6 +160,7 @@ def test_2_group_schema(self):
             'is_organization': False,
             'type': u'group',
             'image_url': u'',
+            'image_display_url': u'',
             'packages': sorted([{'id': group_pack[0].id,
                                  'name': group_pack[0].name,
                                  'title': group_pack[0].title},

From fc1097763c8d9bdb3be42bbd43c037aab59f1b6d Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Thu, 10 Oct 2013 20:55:23 +0100
Subject: [PATCH 38/87] [#1267] fix url for by faking routes threadlocal

---
 ckan/lib/cli.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py
index 14c217ed096..5d13da6638b 100644
--- a/ckan/lib/cli.py
+++ b/ckan/lib/cli.py
@@ -10,6 +10,8 @@
 import ckan.include.rcssmin as rcssmin
 import ckan.lib.fanstatic_resources as fanstatic_resources
 import sqlalchemy as sa
+import urlparse
+import routes
 
 import paste.script
 from paste.registry import Registry
@@ -99,6 +101,12 @@ def _load_config(self):
         self.translator_obj = MockTranslator()
         self.registry.register(pylons.translator, self.translator_obj)
 
+        ## give routes enough information to run url_for
+        parsed = urlparse.urlparse(conf.get('ckan.site_url', 'http://0.0.0.0'))
+        request_config = routes.request_config()
+        request_config.host = parsed.netloc + parsed.path
+        request_config.protocol = parsed.scheme
+
     def _setup_app(self):
         cmd = paste.script.appinstall.SetupCommand('setup-app')
         cmd.run([self.filename])

From 5023e0c3354ca03475b06ee11a3e47f867adedaf Mon Sep 17 00:00:00 2001
From: Nigel Babu <nigelbabu@gmail.com>
Date: Mon, 14 Oct 2013 10:51:01 +0530
Subject: [PATCH 39/87] Don't exempt sysadmins

---
 ckan/logic/auth/create.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py
index ec38483bd0d..ca3a2d70b6d 100644
--- a/ckan/logic/auth/create.py
+++ b/ckan/logic/auth/create.py
@@ -104,6 +104,7 @@ def rating_create(context, data_dict):
     return {'success': True}
 
 
+@logic.auth_sysadmins_check
 def user_create(context, data_dict=None):
     using_api = 'api_version' in context
     create_user_via_api = new_authz.check_config_permission(

From 878772cfc37932805cca58439150d702506fe043 Mon Sep 17 00:00:00 2001
From: Nigel Babu <nigelbabu@gmail.com>
Date: Mon, 14 Oct 2013 12:22:29 +0530
Subject: [PATCH 40/87] [#1226] Add tests

---
 ckan/tests/functional/api/test_user.py | 190 ++++++++++++++++++++++++-
 ckan/tests/logic/test_auth.py          |   3 +-
 2 files changed, 189 insertions(+), 4 deletions(-)

diff --git a/ckan/tests/functional/api/test_user.py b/ckan/tests/functional/api/test_user.py
index 06dd35dd7fc..26565810c2f 100644
--- a/ckan/tests/functional/api/test_user.py
+++ b/ckan/tests/functional/api/test_user.py
@@ -1,20 +1,26 @@
+import paste
+from pylons import config
 from nose.tools import assert_equal
 
 import ckan.logic as logic
+import ckan.new_authz as new_authz
 from ckan import model
 from ckan.lib.create_test_data import CreateTestData
 from ckan.tests import TestController as ControllerTestCase
+from ckan.tests.pylons_controller import PylonsTestCase
 from ckan.tests import url_for
+import ckan.config.middleware
+from ckan.common import json
 
 class TestUserApi(ControllerTestCase):
     @classmethod
     def setup_class(cls):
         CreateTestData.create()
-                
+
     @classmethod
     def teardown_class(cls):
         model.repo.rebuild_db()
-        
+
     def test_autocomplete(self):
         response = self.app.get(
             url=url_for(controller='api', action='user_autocomplete', ver=2),
@@ -51,8 +57,186 @@ def test_autocomplete_limit(self):
         print response.json
         assert_equal(len(response.json), 1)
 
+
+class TestCreateUserApiDisabled(PylonsTestCase):
+    '''
+    Tests for the creating user when create_user_via_api is disabled.
+    '''
+
+    @classmethod
+    def setup_class(cls):
+        CreateTestData.create()
+        cls._original_config = config.copy()
+        new_authz.clear_auth_functions_cache()
+        wsgiapp = ckan.config.middleware.make_app(config['global_conf'],
+                **config)
+        cls.app = paste.fixture.TestApp(wsgiapp)
+        cls.sysadmin_user = model.User.get('testsysadmin')
+        PylonsTestCase.setup_class()
+
+    @classmethod
+    def teardown_class(cls):
+        config.clear()
+        config.update(cls._original_config)
+        new_authz.clear_auth_functions_cache()
+        PylonsTestCase.teardown_class()
+
+        model.repo.rebuild_db()
+
+    def test_user_create_api_disabled_sysadmin(self):
+        params = {
+            'name': 'testinganewusersysadmin',
+            'email': 'testinganewuser@ckan.org',
+            'password': 'random',
+        }
+        res = self.app.post('/api/3/action/user_create', json.dumps(params),
+                extra_environ={'Authorization': str(self.sysadmin_user.apikey)},
+                expect_errors=True)
+        res_dict = res.json
+        assert res_dict['success'] is False
+
+    def test_user_create_api_disabled_anon(self):
+        params = {
+            'name': 'testinganewuseranon',
+            'email': 'testinganewuser@ckan.org',
+            'password': 'random',
+        }
+        res = self.app.post('/api/3/action/user_create', json.dumps(params),
+                expect_errors=True)
+        res_dict = res.json
+        assert res_dict['success'] is False
+
+
+class TestCreateUserApiEnabled(PylonsTestCase):
+    '''
+    Tests for the creating user when create_user_via_api is enabled.
+    '''
+
+    @classmethod
+    def setup_class(cls):
+        CreateTestData.create()
+        cls._original_config = config.copy()
+        config['ckan.auth.create_user_via_api'] = True
+        new_authz.clear_auth_functions_cache()
+        wsgiapp = ckan.config.middleware.make_app(config['global_conf'],
+                **config)
+        cls.app = paste.fixture.TestApp(wsgiapp)
+        PylonsTestCase.setup_class()
+        cls.sysadmin_user = model.User.get('testsysadmin')
+
+    @classmethod
+    def teardown_class(cls):
+        config.clear()
+        config.update(cls._original_config)
+        new_authz.clear_auth_functions_cache()
+        PylonsTestCase.teardown_class()
+
+        model.repo.rebuild_db()
+
+    def test_user_create_api_disabled_sysadmin(self):
+        params = {
+            'name': 'testinganewusersysadmin',
+            'email': 'testinganewuser@ckan.org',
+            'password': 'random',
+        }
+        res = self.app.post('/api/3/action/user_create', json.dumps(params),
+                extra_environ={'Authorization':
+                    str(self.sysadmin_user.apikey)})
+        res_dict = res.json
+        assert res_dict['success'] is True
+
+    def test_user_create_api_disabled_anon(self):
+        params = {
+            'name': 'testinganewuseranon',
+            'email': 'testinganewuser@ckan.org',
+            'password': 'random',
+        }
+        res = self.app.post('/api/3/action/user_create', json.dumps(params),
+                expect_errors=True)
+        res_dict = res.json
+        assert res_dict['success'] is True
+
+
+class TestCreateUserWebDisabled(PylonsTestCase):
+    '''
+    Tests for the creating user by create_user_via_web is disabled.
+    '''
+
+    @classmethod
+    def setup_class(cls):
+        CreateTestData.create()
+        cls._original_config = config.copy()
+        config['ckan.auth.create_user_via_web'] = False
+        new_authz.clear_auth_functions_cache()
+        wsgiapp = ckan.config.middleware.make_app(config['global_conf'],
+                **config)
+        cls.app = paste.fixture.TestApp(wsgiapp)
+        cls.sysadmin_user = model.User.get('testsysadmin')
+        PylonsTestCase.setup_class()
+
+    @classmethod
+    def teardown_class(cls):
+        config.clear()
+        config.update(cls._original_config)
+        new_authz.clear_auth_functions_cache()
+        PylonsTestCase.teardown_class()
+
+        model.repo.rebuild_db()
+
+    def test_user_create_api_disabled(self):
+        params = {
+            'name': 'testinganewuser',
+            'email': 'testinganewuser@ckan.org',
+            'password': 'random',
+        }
+        res = self.app.post('/api/3/action/user_create', json.dumps(params),
+                extra_environ={'Authorization': str(self.sysadmin_user.apikey)},
+                expect_errors=True)
+        res_dict = res.json
+        assert res_dict['success'] is False
+
+
+class TestCreateUserWebEnabled(PylonsTestCase):
+    '''
+    Tests for the creating user by create_user_via_web is enabled.
+    '''
+
+    @classmethod
+    def setup_class(cls):
+        CreateTestData.create()
+        cls._original_config = config.copy()
+        config['ckan.auth.create_user_via_web'] = True
+        new_authz.clear_auth_functions_cache()
+        wsgiapp = ckan.config.middleware.make_app(config['global_conf'],
+                **config)
+        cls.app = paste.fixture.TestApp(wsgiapp)
+        cls.sysadmin_user = model.User.get('testsysadmin')
+        PylonsTestCase.setup_class()
+
+    @classmethod
+    def teardown_class(cls):
+        config.clear()
+        config.update(cls._original_config)
+        new_authz.clear_auth_functions_cache()
+        PylonsTestCase.teardown_class()
+
+        model.repo.rebuild_db()
+
+    def test_user_create_api_disabled(self):
+        params = {
+            'name': 'testinganewuser',
+            'email': 'testinganewuser@ckan.org',
+            'password': 'random',
+        }
+        res = self.app.post('/api/3/action/user_create', json.dumps(params),
+                extra_environ={'Authorization': str(self.sysadmin_user.apikey)},
+                expect_errors=True)
+        res_dict = res.json
+        assert res_dict['success'] is False
+
+
 class TestUserActions(object):
-    
+
     @classmethod
     def setup_class(cls):
         CreateTestData.create()
diff --git a/ckan/tests/logic/test_auth.py b/ckan/tests/logic/test_auth.py
index 3ea9f2db39f..b73834d2657 100644
--- a/ckan/tests/logic/test_auth.py
+++ b/ckan/tests/logic/test_auth.py
@@ -11,8 +11,9 @@
     'user_create_organizations': False,
     'user_delete_groups': False,
     'user_delete_organizations': False,
-    'create_user_via_api': False,
     'create_unowned_dataset': False,
+    'create_user_via_api': False,
+    'create_user_via_web': True,
 }
 
 

From 3273ebf7d0e06dbf3d854a25f701e651fe88bb0b Mon Sep 17 00:00:00 2001
From: Nigel Babu <nigelbabu@gmail.com>
Date: Mon, 14 Oct 2013 15:34:07 +0530
Subject: [PATCH 41/87] [#1226] Return the old behaviour for create_user

---
 ckan/logic/auth/create.py              | 1 -
 ckan/tests/functional/api/test_user.py | 6 ++----
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py
index 13e02125855..53c8ae153ab 100644
--- a/ckan/logic/auth/create.py
+++ b/ckan/logic/auth/create.py
@@ -105,7 +105,6 @@ def rating_create(context, data_dict):
 
 
 @logic.auth_allow_anonymous_access
-@logic.auth_sysadmins_check
 def user_create(context, data_dict=None):
     using_api = 'api_version' in context
     create_user_via_api = new_authz.check_config_permission(
diff --git a/ckan/tests/functional/api/test_user.py b/ckan/tests/functional/api/test_user.py
index ba67b97455e..cdbe039d012 100644
--- a/ckan/tests/functional/api/test_user.py
+++ b/ckan/tests/functional/api/test_user.py
@@ -83,7 +83,7 @@ def teardown_class(cls):
 
         model.repo.rebuild_db()
 
-    def test_user_create_api_disabled_sysadmin(self):
+    def test_user_create_api_enabled_sysadmin(self):
         params = {
             'name': 'testinganewusersysadmin',
             'email': 'testinganewuser@ckan.org',
@@ -93,7 +93,7 @@ def test_user_create_api_disabled_sysadmin(self):
                 extra_environ={'Authorization': str(self.sysadmin_user.apikey)},
                 expect_errors=True)
         res_dict = res.json
-        assert res_dict['success'] is False
+        assert res_dict['success'] is True
 
     def test_user_create_api_disabled_anon(self):
         params = {
@@ -190,7 +190,6 @@ def test_user_create_api_disabled(self):
             'password': 'random',
         }
         res = self.app.post('/api/3/action/user_create', json.dumps(params),
-                extra_environ={'Authorization': str(self.sysadmin_user.apikey)},
                 expect_errors=True)
         res_dict = res.json
         assert res_dict['success'] is False
@@ -229,7 +228,6 @@ def test_user_create_api_disabled(self):
             'password': 'random',
         }
         res = self.app.post('/api/3/action/user_create', json.dumps(params),
-                extra_environ={'Authorization': str(self.sysadmin_user.apikey)},
                 expect_errors=True)
         res_dict = res.json
         assert res_dict['success'] is False

From bb6cb7d0ff417b72f002dd372bdfd09ba560a766 Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Mon, 14 Oct 2013 21:46:39 +0100
Subject: [PATCH 42/87] [#1261] add filesize limit, make storage folder
 backwards compatable, document a bit more

---
 ckan/lib/helpers.py  |  3 +-
 ckan/lib/uploader.py | 69 ++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 66 insertions(+), 6 deletions(-)

diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
index fceab6f1b6a..3aa73ad3a8b 100644
--- a/ckan/lib/helpers.py
+++ b/ckan/lib/helpers.py
@@ -37,6 +37,7 @@
 import ckan.lib.maintain as maintain
 import ckan.lib.datapreview as datapreview
 import ckan.logic as logic
+import ckan.lib.uploader as uploader
 
 from ckan.common import (
     _, ungettext, g, c, request, session, json, OrderedDict
@@ -1664,7 +1665,7 @@ def new_activities():
     return action({}, {})
 
 def uploads_enabled():
-    if config.get('ckan.storage_path'):
+    if uploader.get_storage_path():
         return True
     return False
 
diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py
index eb0f28eec54..880488b8161 100644
--- a/ckan/lib/uploader.py
+++ b/ckan/lib/uploader.py
@@ -3,11 +3,51 @@
 import pylons
 import datetime
 import ckan.lib.munge as munge
+import logging
+import ckan.logic as logic
+
+log = logging.getLogger(__name__)
+
+_storage_path = None
+
+def get_storage_path():
+    '''Function to cache storage path'''
+    global _storage_path
+
+    #None means it has not been set. False means not in config.
+    if _storage_path is None:
+        storage_path = pylons.config.get('ckan.storage_path')
+        ofs_impl = pylons.config.get('ofs.impl')
+        ofs_storage_dir = pylons.config.get('ofs.storage_dir')
+        if storage_path:
+            _storage_path = storage_path
+        elif ofs_impl == 'pairtree' and ofs_storage_dir:
+            log.warn('''Please use config option ckan.storage_path instaed of
+                     ofs.storage_path''')
+            _storage_path = ofs_storage_dir
+            return _storage_path
+        elif ofs_impl:
+            log.critical('''We only support local file storage form version 2.2 of ckan
+                          please specify ckan.storage_path in your config for your uploads''')
+            _storage_path = False
+        else:
+            log.critical('''Please specify a ckan.storage_path in your config for your uploads''')
+            _storage_path = False
+
+    return _storage_path
+
 
 
 class Upload(object):
     def __init__(self, object_type, old_filename=None):
-        path = pylons.config.get('ckan.storage_path', '/tmp')
+        ''' Setup upload by creating  a subdirectory of the storage directory of name
+        object_type. old_filename is the name of the file in the url field last time'''
+        self.storage_path = None
+        self.filename = None
+        self.filepath = None
+        path = get_storage_path()
+        if not path:
+            return
         self.storage_path = os.path.join(path, 'storage',
                                          'uploads', object_type)
         try:
@@ -18,12 +58,20 @@ def __init__(self, object_type, old_filename=None):
         self.old_filename = old_filename
         if old_filename:
             self.old_filepath = os.path.join(self.storage_path, old_filename)
-        self.filename = None
-        self.filepath = None
 
     def update_data_dict(self, data_dict, url_field, file_field, clear_field):
+        ''' Manipulate data from the data_dict.  url_field is the name of the field
+        where the upload is going to be. file_field is name of the key where the
+        FieldStorage is kept (i.e the field where the file data actually is). clear_field
+        is the name of a boolean field which requests the upload to be deleted.  This needs
+        to be called before it reaches any validators'''
+
+        if not self.storage_path:
+            return
+
         self.url = data_dict.get(url_field, '')
         self.clear = data_dict.pop(clear_field, None)
+        self.file_field = file_field
         self.upload_field_storage = data_dict.pop(file_field, None)
 
         if isinstance(self.upload_field_storage, cgi.FieldStorage):
@@ -41,16 +89,27 @@ def update_data_dict(self, data_dict, url_field, file_field, clear_field):
             if self.clear and self.url == self.old_filename:
                 data_dict[url_field] = ''
 
-    def upload(self):
+    def upload(self, max_size=2):
+        ''' Actually upload the file.  This should happen just before a commit but after
+        the data has been validated and flushed to the db.  This is so we do not store anything
+        unless the request is actually good. Max_size is size in MB maximum of the file'''
+
         if self.filename:
             output_file = open(self.tmp_filepath, 'wb')
             self.upload_file.seek(0)
+            current_size = 0
             while True:
-                # mb chuncks
+                current_size = current_size + 1
+                # MB chuncks
                 data = self.upload_file.read(2 ** 20)
                 if not data:
                     break
                 output_file.write(data)
+                if current_size > max_size:
+                    os.remove(self.tmp_filepath)
+                    raise logic.ValidationError(
+                        {self.file_field: ['File upload too large']}
+                    )
             output_file.close()
             os.rename(self.tmp_filepath, self.filepath)
             self.clear = True

From 3dee85226f1fa6a29a03d598c8c11be1c5dba4ca Mon Sep 17 00:00:00 2001
From: John Martin <me@johnmart.in>
Date: Tue, 15 Oct 2013 15:10:09 +0100
Subject: [PATCH 43/87] [#1262] First working version of the JS module for
 image uploading

---
 .../base/javascript/modules/image-upload.js   | 147 ++++++++++++++++++
 ckan/public/base/javascript/resource.config   |   1 +
 ckan/public/base/less/forms.less              |  34 ++++
 ckan/templates/group/snippets/group_form.html |  10 +-
 ckan/templates/macros/form.html               |  27 ++++
 .../snippets/organization_form.html           |   9 +-
 6 files changed, 211 insertions(+), 17 deletions(-)
 create mode 100644 ckan/public/base/javascript/modules/image-upload.js

diff --git a/ckan/public/base/javascript/modules/image-upload.js b/ckan/public/base/javascript/modules/image-upload.js
new file mode 100644
index 00000000000..69ca8f93c05
--- /dev/null
+++ b/ckan/public/base/javascript/modules/image-upload.js
@@ -0,0 +1,147 @@
+/* Image Upload
+ * 
+ *
+ * Options
+ * - id: what's the id of the context?
+ */  
+this.ckan.module('image-upload', function($, _) {
+  return {
+    /* options object can be extended using data-module-* attributes */
+    options: {
+      is_url: true,
+      has_image: false,
+      i18n: {
+        upload: _('From computer'),
+        url: _('From web'),
+        remove: _('Remove'),
+        label: _('Upload image'),
+        label_url: _('Image URL'),
+        remove_tooltip: _('Reset this')
+      },
+      template: [
+        ''
+      ].join("\n")
+    },
+
+    state: {
+      attached: 1,
+      blank: 2,
+      web: 3
+    },
+
+    /* Initialises the module setting up elements and event listeners.
+     *
+     * Returns nothing.
+     */
+    initialize: function () {
+      $.proxyAll(this, /_on/);
+      var options = this.options;
+
+      // firstly setup the fields
+      this.input = $('input[name="image_upload"]', this.el);
+      this.field_url = $('input[name="image_url"]', this.el).parents('.control-group');
+      this.field_image = this.input.parents('.control-group');
+
+      var checkbox = $('input[name="clear_upload"]', this.el);
+      if (checkbox.length > 0) {
+        options.has_image = true;
+        checkbox.parents('.control-group').remove();
+      }
+
+      this.field_clear = $('<input type="hidden" name="clear_upload">')
+        .appendTo(this.el);
+
+      this.button_url = $('<a href="javascript:;" class="btn"><i class="icon-globe"></i> '+this.i18n('url')+'</a>')
+        .on('click', this._onFromWeb)
+        .insertAfter(this.input);
+
+      this.button_upload = $('<a href="javascript:;" class="btn"><i class="icon-cloud-upload"></i>'+this.i18n('upload')+'</a>')
+        .insertAfter(this.input);
+
+      this.button_remove = $('<a href="javascript:;" class="btn btn-danger" />')
+        .text(this.i18n('remove'))
+        .on('click', this._onRemove)
+        .insertAfter(this.button_upload);
+
+      $('<a href="javascript:;" class="btn btn-danger btn-remove-url"><i class="icon-remove"></i></a>')
+        .prop('title', this.i18n('remove_tooltip'))
+        .on('click', this._onRemove)
+        .insertBefore($('input', this.field_url));
+
+      $('label[for="field-image-upload"]').text(this.i18n('label'));
+
+      this.input
+        .on('mouseover', this._onInputMouseOver)
+        .on('mouseout', this._onInputMouseOut)
+        .on('change', this._onInputChange)
+        .css('width', this.button_upload.outerWidth())
+        .hide();
+
+      this.fields = $('<i />')
+        .add(this.button_remove)
+        .add(this.button_upload)
+        .add(this.button_url)
+        .add(this.input)
+        .add(this.field_url)
+        .add(this.field_image);
+
+      if (options.is_url) {
+        this.changeState(this.state.web);
+      } else if (options.has_image) {
+        this.changeState(this.state.attached);
+      } else {
+        this.changeState(this.state.blank);
+      }
+
+    },
+
+    changeState: function(state) {
+      this.fields.hide();
+      if (state == this.state.blank) {
+        this.button_upload
+          .add(this.field_image)
+          .add(this.button_url)
+          .add(this.input)
+          .show();
+      } else if (state == this.state.attached) {
+        this.button_remove
+          .add(this.field_image)
+          .show();
+      } else if (state == this.state.web) {
+        this.field_url
+          .show();
+      }
+    },
+
+    _onFromWeb: function() {
+      this.changeState(this.state.web);
+      $('input', this.field_url).focus();
+      if (this.options.has_image) {
+        this.field_clear.val('true');
+      }
+    },
+
+    _onRemove: function() {
+      this.changeState(this.state.blank);
+      $('input', this.field_url).val('');
+      if (this.options.has_image) {
+        this.field_clear.val('true');
+      }
+    },
+
+    _onInputChange: function() {
+      this.file_name = this.input.val();
+      this.field_clear.val('');
+      this.changeState(this.state.attached);
+    },
+
+    _onInputMouseOver: function() {
+      this.button_upload.addClass('hover');
+    },
+
+    _onInputMouseOut: function() {
+      this.button_upload.removeClass('hover');
+    }
+
+  }
+});
diff --git a/ckan/public/base/javascript/resource.config b/ckan/public/base/javascript/resource.config
index b1a7302d9e8..564da811a6e 100644
--- a/ckan/public/base/javascript/resource.config
+++ b/ckan/public/base/javascript/resource.config
@@ -38,6 +38,7 @@ ckan =
     modules/table-toggle-more.js
     modules/dataset-visibility.js
     modules/media-grid.js
+    modules/image-upload.js
 
 main =
     apply_html_class
diff --git a/ckan/public/base/less/forms.less b/ckan/public/base/less/forms.less
index 289439c291a..2bfc1a2d492 100644
--- a/ckan/public/base/less/forms.less
+++ b/ckan/public/base/less/forms.less
@@ -670,3 +670,37 @@ textarea {
     .box-shadow(none);
   }
 }
+
+.js .image-upload {
+  #field-image-upload {
+    cursor: pointer;
+    position: absolute;
+    z-index: 1;
+    .opacity(0);
+  }
+  .controls {
+    position: relative;
+  }
+  .btn {
+    position: relative;
+    top: 0;
+    margin-right: 10px;
+    &.hover {
+      color: @grayDark;
+      text-decoration: none;
+      background-position: 0 -15px;
+      .transition(background-position .1s linear);
+    }
+  }
+  .btn-remove-url {
+    position: absolute;
+    margin-right: 0;
+    top: 4px;
+    right: 5px;
+    padding: 0 4px;
+    .border-radius(100px);
+    .icon-remove {
+      margin-right: 0;
+    }
+  }
+}
diff --git a/ckan/templates/group/snippets/group_form.html b/ckan/templates/group/snippets/group_form.html
index 322f9ebff8a..cd19912d66b 100644
--- a/ckan/templates/group/snippets/group_form.html
+++ b/ckan/templates/group/snippets/group_form.html
@@ -19,15 +19,7 @@
 
     {{ form.markdown('description', label=_('Description'), id='field-description', placeholder=_('A little information about my group...'), value=data.description, error=errors.description) }}
 
-    {{ form.input('image_url', label=_('Image URL'), id='field-image-url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }}
-
-
-    {% if h.uploads_enabled() %}
-    {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }}
-      {% if data.image_url and not data.image_url.startswith('http') %}
-        {{ form.checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }}
-      {% endif %}
-    {% endif %}
+    {{ form.image_upload(data, errors, image_url=c.group_dict.image_display_url, is_upload_enabled=h.uploads_enabled()) }}
 
   {% endblock %}
 
diff --git a/ckan/templates/macros/form.html b/ckan/templates/macros/form.html
index 497dea6bf02..7dc044ccca2 100644
--- a/ckan/templates/macros/form.html
+++ b/ckan/templates/macros/form.html
@@ -395,3 +395,30 @@
   </p>
 {% endmacro %}
 
+{#
+Builds a file upload for input
+
+Example
+  {% import 'macros/form.html' as form %}
+  {{ form.image_upload(data, errors, is_upload_enabled=true) }}
+
+#}
+{% macro image_upload(data, errors, image_url=false, is_upload_enabled=false, placeholder=false) %}
+  {% set placeholder = placeholder if placeholder else _('http://example.com/my-image.jpg') %}
+  {% set has_uploaded_data = data.image_url and not data.image_url.startswith('http') %}
+  {% set is_url = data.image_url and data.image_url.startswith('http') %}
+
+  {% if is_upload_enabled %}<div class="image-upload" data-module="image-upload" data-module-is_url="{{ 'true' if is_url else 'false' }}" data-module-image_url="{{ image_url }}">{% endif %}
+
+  {{ input('image_url', label=_('Image URL'), id='field-image-url', placeholder=placeholder, value=data.image_url, error=errors.image_url, classes=['control-full']) }}
+
+  {% if is_upload_enabled %}
+    {{ input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }}
+    {% if has_uploaded_data %}
+      {{ checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }}
+    {% endif %}
+  {% endif %}
+
+  {% if is_upload_enabled %}</div>{% endif %}
+
+{% endmacro %}
diff --git a/ckan/templates/organization/snippets/organization_form.html b/ckan/templates/organization/snippets/organization_form.html
index 0a2134e89f7..0b97b958d38 100644
--- a/ckan/templates/organization/snippets/organization_form.html
+++ b/ckan/templates/organization/snippets/organization_form.html
@@ -19,14 +19,7 @@
 
     {{ form.markdown('description', label=_('Description'), id='field-description', placeholder=_('A little information about my organization...'), value=data.description, error=errors.description) }}
 
-    {{ form.input('image_url', label=_('Image URL'), id='field-image-url', placeholder=_('http://example.com/my-image.jpg'), value=data.image_url, error=errors.image_url, classes=['control-full']) }}
-
-    {% if h.uploads_enabled() %}
-    {{ form.input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value=data.image_url, error='', classes=['control-full']) }}
-      {% if data.image_url and not data.image_url.startswith('http') %}
-        {{ form.checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }}
-      {% endif %}
-    {% endif %}
+    {{ form.image_upload(data, errors, image_url=c.group_dict.image_display_url, is_upload_enabled=h.uploads_enabled()) }}
 
   {% endblock %}
 

From 0f74db8a964451e5cba1360b4f15ea3a84a56544 Mon Sep 17 00:00:00 2001
From: John Martin <me@johnmart.in>
Date: Tue, 15 Oct 2013 15:21:14 +0100
Subject: [PATCH 44/87] [#1262] Adds in doc blocks into the image_upload JS
 module

---
 .../base/javascript/modules/image-upload.js   | 46 +++++++++++++++++--
 1 file changed, 41 insertions(+), 5 deletions(-)

diff --git a/ckan/public/base/javascript/modules/image-upload.js b/ckan/public/base/javascript/modules/image-upload.js
index 69ca8f93c05..dbb83ade15a 100644
--- a/ckan/public/base/javascript/modules/image-upload.js
+++ b/ckan/public/base/javascript/modules/image-upload.js
@@ -1,8 +1,5 @@
 /* Image Upload
  * 
- *
- * Options
- * - id: what's the id of the context?
  */  
 this.ckan.module('image-upload', function($, _) {
   return {
@@ -42,41 +39,49 @@ this.ckan.module('image-upload', function($, _) {
       this.field_url = $('input[name="image_url"]', this.el).parents('.control-group');
       this.field_image = this.input.parents('.control-group');
 
+      // Is there a clear checkbox on the form already?
       var checkbox = $('input[name="clear_upload"]', this.el);
       if (checkbox.length > 0) {
         options.has_image = true;
         checkbox.parents('.control-group').remove();
       }
 
+      // Adds the hidden clear input to the form
       this.field_clear = $('<input type="hidden" name="clear_upload">')
         .appendTo(this.el);
 
+      // Button to set the field to be a URL
       this.button_url = $('<a href="javascript:;" class="btn"><i class="icon-globe"></i> '+this.i18n('url')+'</a>')
         .on('click', this._onFromWeb)
         .insertAfter(this.input);
 
+      // Button to attach local file to the form
       this.button_upload = $('<a href="javascript:;" class="btn"><i class="icon-cloud-upload"></i>'+this.i18n('upload')+'</a>')
         .insertAfter(this.input);
 
+      // Button to reset the form back to the first from when there is a image uploaded
       this.button_remove = $('<a href="javascript:;" class="btn btn-danger" />')
         .text(this.i18n('remove'))
         .on('click', this._onRemove)
         .insertAfter(this.button_upload);
 
+      // Button for resetting the form when there is a URL set
       $('<a href="javascript:;" class="btn btn-danger btn-remove-url"><i class="icon-remove"></i></a>')
         .prop('title', this.i18n('remove_tooltip'))
         .on('click', this._onRemove)
         .insertBefore($('input', this.field_url));
 
+      // Update the main label
       $('label[for="field-image-upload"]').text(this.i18n('label'));
 
+      // Setup the file input
       this.input
         .on('mouseover', this._onInputMouseOver)
         .on('mouseout', this._onInputMouseOut)
         .on('change', this._onInputChange)
-        .css('width', this.button_upload.outerWidth())
-        .hide();
+        .css('width', this.button_upload.outerWidth());
 
+      // Fields storage. Used in this.changeState
       this.fields = $('<i />')
         .add(this.button_remove)
         .add(this.button_upload)
@@ -85,6 +90,7 @@ this.ckan.module('image-upload', function($, _) {
         .add(this.field_url)
         .add(this.field_image);
 
+      // Setup the initial state
       if (options.is_url) {
         this.changeState(this.state.web);
       } else if (options.has_image) {
@@ -95,6 +101,16 @@ this.ckan.module('image-upload', function($, _) {
 
     },
 
+    /* Method to change the display state of the image fields
+     *
+     * state - Pseudo constant for passing the state we should be in now
+     *
+     * Examples
+     *
+     *   this.changeState(this.state.web); // Sets the state in URL mode
+     *
+     * Returns nothing.
+     */
     changeState: function(state) {
       this.fields.hide();
       if (state == this.state.blank) {
@@ -113,6 +129,10 @@ this.ckan.module('image-upload', function($, _) {
       }
     },
 
+    /* Event listener for when someone sets the field to URL mode
+     *
+     * Returns nothing.
+     */
     _onFromWeb: function() {
       this.changeState(this.state.web);
       $('input', this.field_url).focus();
@@ -121,6 +141,10 @@ this.ckan.module('image-upload', function($, _) {
       }
     },
 
+    /* Event listener for resetting the field back to the blank state
+     *
+     * Returns nothing.
+     */
     _onRemove: function() {
       this.changeState(this.state.blank);
       $('input', this.field_url).val('');
@@ -129,16 +153,28 @@ this.ckan.module('image-upload', function($, _) {
       }
     },
 
+    /* Event listener for when someone chooses a file to upload
+     *
+     * Returns nothing.
+     */
     _onInputChange: function() {
       this.file_name = this.input.val();
       this.field_clear.val('');
       this.changeState(this.state.attached);
     },
 
+    /* Event listener for when a user mouseovers the hidden file input
+     *
+     * Returns nothing.
+     */
     _onInputMouseOver: function() {
       this.button_upload.addClass('hover');
     },
 
+    /* Event listener for when a user mouseouts the hidden file input
+     *
+     * Returns nothing.
+     */
     _onInputMouseOut: function() {
       this.button_upload.removeClass('hover');
     }

From 7d15d591499d5722f60cc0ecfc09abf59dad177f Mon Sep 17 00:00:00 2001
From: John Martin <me@johnmart.in>
Date: Tue, 15 Oct 2013 15:29:45 +0100
Subject: [PATCH 45/87] [#1262] Adds support for different field names for
 image upload

---
 ckan/public/base/javascript/modules/image-upload.js | 13 +++++++------
 ckan/templates/macros/form.html                     |  8 ++++----
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/ckan/public/base/javascript/modules/image-upload.js b/ckan/public/base/javascript/modules/image-upload.js
index dbb83ade15a..96308ed5764 100644
--- a/ckan/public/base/javascript/modules/image-upload.js
+++ b/ckan/public/base/javascript/modules/image-upload.js
@@ -7,6 +7,9 @@ this.ckan.module('image-upload', function($, _) {
     options: {
       is_url: true,
       has_image: false,
+      field_upload: 'input[name="image_upload"]',
+      field_url: 'input[name="image_url"]',
+      field_clear: 'input[name="clear_upload"]',
       i18n: {
         upload: _('From computer'),
         url: _('From web'),
@@ -35,12 +38,12 @@ this.ckan.module('image-upload', function($, _) {
       var options = this.options;
 
       // firstly setup the fields
-      this.input = $('input[name="image_upload"]', this.el);
-      this.field_url = $('input[name="image_url"]', this.el).parents('.control-group');
+      this.input = $(options.field_upload, this.el);
+      this.field_url = $(options.field_url, this.el).parents('.control-group');
       this.field_image = this.input.parents('.control-group');
 
       // Is there a clear checkbox on the form already?
-      var checkbox = $('input[name="clear_upload"]', this.el);
+      var checkbox = $(options.field_clear, this.el);
       if (checkbox.length > 0) {
         options.has_image = true;
         checkbox.parents('.control-group').remove();
@@ -148,9 +151,7 @@ this.ckan.module('image-upload', function($, _) {
     _onRemove: function() {
       this.changeState(this.state.blank);
       $('input', this.field_url).val('');
-      if (this.options.has_image) {
-        this.field_clear.val('true');
-      }
+      this.field_clear.val('true');
     },
 
     /* Event listener for when someone chooses a file to upload
diff --git a/ckan/templates/macros/form.html b/ckan/templates/macros/form.html
index 7dc044ccca2..5a6c00a6cb1 100644
--- a/ckan/templates/macros/form.html
+++ b/ckan/templates/macros/form.html
@@ -403,19 +403,19 @@
   {{ form.image_upload(data, errors, is_upload_enabled=true) }}
 
 #}
-{% macro image_upload(data, errors, image_url=false, is_upload_enabled=false, placeholder=false) %}
+{% macro image_upload(data, errors, field_url='image_url', field_upload='image_upload', field_clear='clear_upload', image_url=false, is_upload_enabled=false, placeholder=false) %}
   {% set placeholder = placeholder if placeholder else _('http://example.com/my-image.jpg') %}
   {% set has_uploaded_data = data.image_url and not data.image_url.startswith('http') %}
   {% set is_url = data.image_url and data.image_url.startswith('http') %}
 
   {% if is_upload_enabled %}<div class="image-upload" data-module="image-upload" data-module-is_url="{{ 'true' if is_url else 'false' }}" data-module-image_url="{{ image_url }}">{% endif %}
 
-  {{ input('image_url', label=_('Image URL'), id='field-image-url', placeholder=placeholder, value=data.image_url, error=errors.image_url, classes=['control-full']) }}
+  {{ input(field_url, label=_('Image URL'), id='field-image-url', placeholder=placeholder, value=data.image_url, error=errors.image_url, classes=['control-full']) }}
 
   {% if is_upload_enabled %}
-    {{ input('image_upload', label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }}
+    {{ input(field_upload, label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }}
     {% if has_uploaded_data %}
-      {{ checkbox('clear_upload', label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }}
+      {{ checkbox(field_clear, label=_('Clear Upload'), id='field-clear-upload', value='true', error='', classes=['control-full']) }}
     {% endif %}
   {% endif %}
 

From 14a3808c013c8654b96a7ee33e4b6c7f5f5a034e Mon Sep 17 00:00:00 2001
From: John Martin <me@johnmart.in>
Date: Wed, 9 Oct 2013 16:04:13 +0100
Subject: [PATCH 46/87] [#1200] Quick tidy up of the resource_edit based
 templates

---
 ckan/config/routing.py                        |  7 ++-
 ckan/templates/package/base_form_page.html    |  5 +-
 ckan/templates/package/edit.html              | 26 ++++----
 ckan/templates/package/edit_base.html         | 15 +++++
 ckan/templates/package/new_resource.html      | 15 -----
 ckan/templates/package/read_base.html         | 15 +----
 ckan/templates/package/resource_data.html     | 60 +++++++++----------
 ckan/templates/package/resource_edit.html     | 16 +----
 .../templates/package/resource_edit_base.html | 24 ++++++++
 ckan/templates/package/snippets/info.html     | 38 ++++++------
 ckanext/datapusher/plugin.py                  |  2 +-
 11 files changed, 110 insertions(+), 113 deletions(-)
 create mode 100644 ckan/templates/package/edit_base.html
 create mode 100644 ckan/templates/package/resource_edit_base.html

diff --git a/ckan/config/routing.py b/ckan/config/routing.py
index 6b41d8a66f2..6366b9ea01c 100644
--- a/ckan/config/routing.py
+++ b/ckan/config/routing.py
@@ -222,7 +222,6 @@ def make_map():
                   ])))
         m.connect('/dataset/{action}/{id}',
                   requirements=dict(action='|'.join([
-                      'edit',
                       'new_metadata',
                       'new_resource',
                       'history',
@@ -234,6 +233,10 @@ def make_map():
                       'delete',
                       'api_data',
                   ])))
+        m.connect('dataset_edit', '/dataset/edit/{id}', action='edit',
+                  ckan_icon='edit')
+        m.connect('dataset_resources', '/dataset/resources/{id}',
+                  action='resources', ckan_icon='time')
         m.connect('dataset_followers', '/dataset/followers/{id}',
                   action='followers', ckan_icon='group')
         m.connect('dataset_activity', '/dataset/activity/{id}',
@@ -247,7 +250,7 @@ def make_map():
         m.connect('/dataset/{id}/resource_delete/{resource_id}',
                   action='resource_delete')
         m.connect('resource_edit', '/dataset/{id}/resource_edit/{resource_id}',
-                  action='resource_edit')
+                  action='resource_edit', ckan_icon='edit')
         m.connect('/dataset/{id}/resource/{resource_id}/download',
                   action='resource_download')
         m.connect('/dataset/{id}/resource/{resource_id}/embed',
diff --git a/ckan/templates/package/base_form_page.html b/ckan/templates/package/base_form_page.html
index 049d83b1ebb..176e2cd8e04 100644
--- a/ckan/templates/package/base_form_page.html
+++ b/ckan/templates/package/base_form_page.html
@@ -2,8 +2,11 @@
 
 {% block primary_content %}
   <section class="module">
+    {% block page_header %}{% endblock %}
     <div class="module-content">
-      {% block form %}{{ c.form | safe }}{% endblock %}
+      {% block primary_content_inner %}
+        {% block form %}{{ c.form | safe }}{% endblock %}
+      {% endblock %}
     </div>
   </section>
 {% endblock %}
diff --git a/ckan/templates/package/edit.html b/ckan/templates/package/edit.html
index 4f15985d033..39229cdd322 100644
--- a/ckan/templates/package/edit.html
+++ b/ckan/templates/package/edit.html
@@ -1,9 +1,22 @@
-{% extends 'package/base_form_page.html' %}
+{% extends 'package/base.html' %}
 
 {% set pkg = c.pkg_dict %}
 
 {% block breadcrumb_content_selected %}{% endblock %}
 
+{% block content_action %}
+  {% link_for _('View dataset'), controller='package', action='read', id=pkg.name, class_='btn', icon='eye-open' %}
+{% endblock %}
+
+{% block content_primary_nav %}
+  {{ h.build_nav_icon('dataset_edit', _('Edit'), id=pkg.name) }}
+  {{ h.build_nav_icon('dataset_resource', _('Resources'), id=pkg.name) }}
+{% endblock %}
+
+{% block primary_content_inner %}
+  {% block form %}{{ c.form | safe }}{% endblock %}
+{% endblock %}
+
 {% block breadcrumb_content %}
   {{ super() }}
   <li class="active">{% link_for _('Edit'), controller='package', action='edit', id=pkg.name %}</li>
@@ -13,16 +26,7 @@
   {% snippet 'package/snippets/resources.html', pkg=pkg, action='resource_edit' %}
 {% endblock %}
 
-{% block content_action %}
-  {% link_for _('View dataset'), controller='package', action='read', id=pkg.name, class_='btn', icon='eye-open' %}
-{% endblock %}
-
 {% block secondary_content %}
-  {% snippet 'package/snippets/info.html', pkg=pkg, action='package_edit' %}
-{% endblock %}
-
-{% block primary_content %}
-  <header class="module-content page-header"></header>
-  {{ super() }}
+  {% snippet 'package/snippets/info.html', pkg=pkg, show_resources=true, hide_follow_button=true %}
 {% endblock %}
 
diff --git a/ckan/templates/package/edit_base.html b/ckan/templates/package/edit_base.html
new file mode 100644
index 00000000000..37462163de3
--- /dev/null
+++ b/ckan/templates/package/edit_base.html
@@ -0,0 +1,15 @@
+{% extends 'package/base.html' %}
+
+{% set pkg = c.pkg_dict %}
+{% set pkg_dict = c.pkg_dict %}
+
+{% block breadcrumb_content_selected %}{% endblock %}
+
+{% block breadcrumb_content %}
+  {{ super() }}
+  <li class="active">{% link_for _('Edit'), controller='package', action='edit', id=pkg.name %}</li>
+{% endblock %}
+
+{% block secondary_content %}
+  {% snippet 'package/snippets/info.html', pkg=pkg, hide_follow_button=true %}
+{% endblock %}
diff --git a/ckan/templates/package/new_resource.html b/ckan/templates/package/new_resource.html
index 7eedbad761f..fa10fb463be 100644
--- a/ckan/templates/package/new_resource.html
+++ b/ckan/templates/package/new_resource.html
@@ -27,18 +27,3 @@ <h2 class="module-heading"><i class="icon-info-sign"></i> {{ _('What\'s a resour
   {{ super() }}
   {% resource 'vendor/fileupload' %}
 {% endblock %}
-
-{% block primary_content %}
-  {% set res = c.resource %}
-  {% if pkg_dict and pkg_dict.state != 'draft' and res %}
-    <header class="module-content page-header">
-      <ul class="nav nav-tabs">
-      {{ h.build_nav_icon('resource_edit', _('Edit Resource'), id=pkg_dict.id, resource_id=res.id) }}
-      {% if 'datapusher' in g.plugins %}
-          {{ h.build_nav_icon('resource_data', _('Resource Data'), id=pkg_dict.id, resource_id=res.id) }}
-      {% endif %}
-      </ul>
-    </header>
-  {% endif %}
-  {{ super() }}
-{% endblock %}
diff --git a/ckan/templates/package/read_base.html b/ckan/templates/package/read_base.html
index daecd18bf75..820b8def6d1 100644
--- a/ckan/templates/package/read_base.html
+++ b/ckan/templates/package/read_base.html
@@ -50,20 +50,7 @@
   {% block secondary_help_content %}{% endblock %}
 
   {% block package_info %}
-    <div class="module context-info">
-      <div class="module-content">
-        <h1 class="heading">{{ pkg.title or pkg.name }}</h1>
-        <div class="nums">
-          <dl>
-            <dt>{{ _('Followers') }}</dt>
-            <dd>{{ h.SI_number_span(h.get_action('dataset_follower_count', {'id': pkg.id})) }}</dd>
-          </dl>
-        </div>
-        <div class="follow_button">
-          {{ h.follow_button('dataset', pkg.name) }}
-        </div>
-      </div>
-    </div>
+    {% snippet "package/snippets/info.html", pkg=pkg %}
   {% endblock %}
 
   {% block package_organization %}
diff --git a/ckan/templates/package/resource_data.html b/ckan/templates/package/resource_data.html
index 30c9fe59e98..1179ed3bfec 100644
--- a/ckan/templates/package/resource_data.html
+++ b/ckan/templates/package/resource_data.html
@@ -1,45 +1,39 @@
-{% extends "package/new_resource.html" %}
+{% extends "package/resource_edit_base.html" %}
 
-{% set pkg_dict = c.pkg_dict %}
-{% set res = c.resource %}
+{% block subtitle %}{{ h.dataset_display_name(pkg) }} - {{ h.resource_display_name(res) }}{% endblock %}
 
-{% block subtitle %}{{ h.dataset_display_name(pkg_dict) }} - {{ h.resource_display_name(res) }}{% endblock %}
+{% block primary_content_inner %}
 
-{% block breadcrumb_content_selected %}{% endblock %}
+  {% set action = h.url_for(controller='ckanext.datapusher.plugin:ResourceDataController', action='resource_data', id=pkg.name, resource_id=res.id) %}
 
-{% block breadcrumb_content %}
-  {{ super() }}
-  <li class="active"><a href="">{{ _('Edit') }}</a></li>
-{% endblock %}
+  <form method="post" action="{{ action }}" >
 
-{% block content_action %}
-  {% link_for _('View resource'), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id, class_='btn', icon='eye-open' %}
-{% endblock %}
+    <button class="btn btn-primary" name="save" type="submit">
+      {{ _('Upload Data') }}
+    </button>
 
-{# logged_in is defined in new_resource.html #}
-{% block form %}
+    <h3>
+      {{ _('Status:') }}
+      {{ status.status.capitalize() if status.status else _('Not Uploaded Yet') }}
+    </h3>
 
-   {% set action = h.url_for(controller='ckanext.datapusher.plugin:ResourceDataController', action='resource_data', id=pkg_dict.id, resource_id=res.id) %}
+    {% if status.error and status.error.message %}
+      <div class="alert alert-error">
+        <strong>{{ _('Upload Error') }}</strong> &mdash; {{ status.error.message }}
+      </div>
+    {% endif %}
 
-   <form class="dataset-form dataset-resource-form form-horizontal" method="post" action="{{ action }}" >
-      <h3> {{ _('Last Upload Status:') }} {% if status.status %} {{ status.status.capitalize() }} {% else %} {{ _('Not Uploaded Yet') }}  {% endif %} </h3>
-      {% if status.error and status.error.message %}  
-        <h3>  {{ _('Upload Error:') }} {{ status.error.message }} </h3>
-      {% endif %} 
-      {% if status.status %}  
-        <h3>   {{ _('Last Updated: ') }} {{ h.time_ago_from_timestamp(status.last_updated) }} </h3>
-      {% endif %} 
+    {% if status.status %}
+      <h3>{{ _('Last Updated: ') }} {{ h.time_ago_from_timestamp(status.last_updated) }} </h3>
+    {% endif %}
 
-      <button class="btn" name="save" value="go-dataset" type="submit">{{ _('Upload Data') }}</button>
-      {% if status.status and status.task_info %}  
-        <h3>   {{ _('Upload Log ') }} </h3>
-        {{ h.debug_inspect(status.task_info.logs) }}
-      {% endif %} 
-      {{ status }}
-   </form>
+    {% if status.status and status.task_info %}
+      <h3>{{ _('Upload Log') }}</h3>
+      {{ h.debug_inspect(status.task_info.logs) }}
+    {% endif %} 
+    
+    {{ status }}
 
-{% endblock %}
+  </form>
 
-{% block secondary_content %}
-  {% snippet 'package/snippets/info.html', pkg=pkg_dict, active=pkg_dict.id, action='resource_edit' %}
 {% endblock %}
diff --git a/ckan/templates/package/resource_edit.html b/ckan/templates/package/resource_edit.html
index 52ac2ab4722..f5667f97350 100644
--- a/ckan/templates/package/resource_edit.html
+++ b/ckan/templates/package/resource_edit.html
@@ -1,21 +1,7 @@
-{% extends "package/new_resource.html" %}
-
-{% set pkg_dict = c.pkg_dict %}
-{% set res = c.resource %}
+{% extends "package/resource_edit_base.html" %}
 
 {% block subtitle %}{{ h.dataset_display_name(pkg_dict) }} - {{ h.resource_display_name(res) }}{% endblock %}
 
-{% block breadcrumb_content_selected %}{% endblock %}
-
-{% block breadcrumb_content %}
-  {{ super() }}
-  <li class="active"><a href="">{{ _('Edit') }}</a></li>
-{% endblock %}
-
-{% block content_action %}
-  {% link_for _('View resource'), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id, class_='btn', icon='eye-open' %}
-{% endblock %}
-
 {# logged_in is defined in new_resource.html #}
 {% block form %}{{ h.snippet('package/snippets/resource_edit_form.html', data=data, errors=errors, error_summary=error_summary, pkg_name=pkg_dict.name, form_action=c.form_action, allow_upload=g.ofs_impl and logged_in) }}{% endblock %}
 
diff --git a/ckan/templates/package/resource_edit_base.html b/ckan/templates/package/resource_edit_base.html
new file mode 100644
index 00000000000..4608f9f2cf9
--- /dev/null
+++ b/ckan/templates/package/resource_edit_base.html
@@ -0,0 +1,24 @@
+{% extends "package/edit_base.html" %}
+
+{% set logged_in = true if c.userobj else false %}
+{% set res = c.resource %}
+
+{% block content_primary_nav %}
+  {{ h.build_nav_icon('resource_edit', _('Edit Resource'), id=pkg.name, resource_id=res.id) }}
+  {% if 'datapusher' in g.plugins %}
+    {{ h.build_nav_icon('resource_data', _('Resource Data'), id=pkg.name, resource_id=res.id) }}
+  {% endif %}
+{% endblock %}
+
+{% block content_action %}
+  {% link_for _('View resource'), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id, class_='btn', icon='eye-open' %}
+{% endblock %}
+
+{% block primary_content_inner %}
+  {% block form %}{% endblock %}
+{% endblock %}
+
+{% block scripts %}
+  {{ super() }}
+  {% resource 'vendor/fileupload' %}
+{% endblock %}
diff --git a/ckan/templates/package/snippets/info.html b/ckan/templates/package/snippets/info.html
index 704cc8225bf..6bcde9137f2 100644
--- a/ckan/templates/package/snippets/info.html
+++ b/ckan/templates/package/snippets/info.html
@@ -2,33 +2,29 @@
 Displays a sidebard module with information for given package
 
 pkg    - The package dict that owns the resources.
-active - The active resource.
-action - The action that this is coming from.
 
 Example:
 
   {% snippet "package/snippets/info.html", pkg=pkg %}
 
 #}
-{% if pkg and h.check_access('package_update', {'id':pkg.id }) %}
+{% if pkg %}
   <section class="module module-narrow">
-    <h2 class="module-heading"><i class="icon-sitemap"></i> {{ _("Edit Dataset") }}</h2>
-    <ul class="unstyled nav nav-simple">
-      <li class="nav-item{% if action == 'package_edit' %} active{% endif %}">
-        {% link_for h.dataset_display_name(pkg)|truncate(30), controller='package', action='edit', id=pkg.name %}
-      </li>
-    </ul>
-    {% set resources = pkg.resources or [] %}
-    <h2 class="module-heading"><i class="icon-reorder"></i> {{ _("Edit Resources") }}</h2>
-    <ul class="unstyled nav nav-simple">
-      {% block package_resource_list %}
-      {% for resource in resources %}
-        <li class="nav-item{{ ' active' if active == resource.id }}">
-          {% link_for h.resource_display_name(resource)|truncate(30), controller='package', action='resource_edit', id=pkg.name, resource_id=resource.id, inner_span=true %}
-        </li>
-      {% endfor %}
-      {% endblock %}
-      <li class="nav-btn"><a href="{{ h.url_for(controller='package', action='new_resource', id=pkg.name)  }}" class="btn{% if action == 'resource_new' %} disabled{% endif %}"><i class="icon-plus"></i> {{ _('Add New Resource') }}</a></li>
-    </ul>
+    <div class="module context-info">
+      <div class="module-content">
+        <h1 class="heading">{{ pkg.title or pkg.name }}</h1>
+        <div class="nums">
+          <dl>
+            <dt>{{ _('Followers') }}</dt>
+            <dd>{{ h.SI_number_span(h.get_action('dataset_follower_count', {'id': pkg.id})) }}</dd>
+          </dl>
+        </div>
+        {% if not hide_follow_button %}
+          <div class="follow_button">
+            {{ h.follow_button('dataset', pkg.name) }}
+          </div>
+        {% endif %}
+      </div>
+    </div>
   </section>
 {% endif %}
\ No newline at end of file
diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py
index 2138497ad57..e6fa45fa61f 100644
--- a/ckanext/datapusher/plugin.py
+++ b/ckanext/datapusher/plugin.py
@@ -119,7 +119,7 @@ def before_map(self, m):
         m.connect(
             'resource_data', '/dataset/{id}/resource_data/{resource_id}',
             controller='ckanext.datapusher.plugin:ResourceDataController',
-            action='resource_data')
+            action='resource_data', ckan_icon='cloud-upload')
         return m
 
     def get_actions(self):

From 99a677e952d7272ae4c30d892b5530b205ccba18 Mon Sep 17 00:00:00 2001
From: Nigel Babu <nigelbabu@gmail.com>
Date: Wed, 16 Oct 2013 11:29:19 +0530
Subject: [PATCH 47/87] Handle the case where the group doesn't exist

---
 ckan/new_authz.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ckan/new_authz.py b/ckan/new_authz.py
index abeb8f9b57a..d9c544677a3 100644
--- a/ckan/new_authz.py
+++ b/ckan/new_authz.py
@@ -235,7 +235,10 @@ def has_user_permission_for_group_or_org(group_id, user_name, permission):
     ''' Check if the user has the given permission for the group '''
     if not group_id:
         return False
-    group_id = model.Group.get(group_id).id
+    group = model.Group.get(group_id)
+    if not group:
+        return False
+    group_id = group.id
 
     # Sys admins can do anything
     if is_sysadmin(user_name):

From 1b451fe28a89e9a8239ad0c7dd0fec800d4508dd Mon Sep 17 00:00:00 2001
From: Nigel Babu <nigelbabu@gmail.com>
Date: Wed, 16 Oct 2013 11:29:40 +0530
Subject: [PATCH 48/87] [#1257] Check for permissions against owner_org

Fix tests that were passing because they depended on this breakage.
---
 ckan/logic/auth/create.py     |  2 +-
 ckan/tests/logic/test_auth.py | 15 +++++++++------
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py
index a18485c2c57..f999803ab54 100644
--- a/ckan/logic/auth/create.py
+++ b/ckan/logic/auth/create.py
@@ -23,7 +23,7 @@ def package_create(context, data_dict=None):
 
     # If an organization is given are we able to add a dataset to it?
     data_dict = data_dict or {}
-    org_id = data_dict.get('organization_id')
+    org_id = data_dict.get('owner_org')
     if org_id and not new_authz.has_user_permission_for_group_or_org(
             org_id, user, 'create_dataset'):
         return {'success': False, 'msg': _('User %s not authorized to add dataset to this organization') % user}
diff --git a/ckan/tests/logic/test_auth.py b/ckan/tests/logic/test_auth.py
index 3ea9f2db39f..10d24e0105a 100644
--- a/ckan/tests/logic/test_auth.py
+++ b/ckan/tests/logic/test_auth.py
@@ -86,17 +86,20 @@ def test_03_create_dataset_no_org(self):
         self._call_api('package_create', dataset, 'no_org', 403)
 
     def test_04_create_dataset_with_org(self):
-
+        org_with_user = self._call_api('organization_show', {'id':
+            'org_with_user'}, 'sysadmin')
         dataset = {'name': 'admin_create_with_user',
-                   'owner_org': 'org_with_user'}
+                   'owner_org': org_with_user.json['result']['id']}
         self._call_api('package_create', dataset, 'sysadmin', 200)
 
+        org_no_user = self._call_api('organization_show', {'id':
+            'org_no_user'}, 'sysadmin')
         dataset = {'name': 'sysadmin_create_no_user',
-                   'owner_org': 'org_no_user'}
+                   'owner_org': org_no_user.json['result']['id']}
         self._call_api('package_create', dataset, 'sysadmin', 200)
 
         dataset = {'name': 'user_create_with_org',
-                   'owner_org': 'org_with_user'}
+                   'owner_org': org_with_user.json['result']['id']}
         self._call_api('package_create', dataset, 'no_org', 403)
 
     def test_05_add_users_to_org(self):
@@ -127,7 +130,7 @@ def _add_datasets(self, user):
 
         #not able to add dataset to org admin does not belong to.
         dataset = {'name': user + '_dataset_bad', 'owner_org': 'org_no_user'}
-        self._call_api('package_create', dataset, user, 409)
+        self._call_api('package_create', dataset, user, 403)
 
         #admin not able to make dataset not owned by a org
         dataset = {'name': user + '_dataset_bad'}
@@ -135,7 +138,7 @@ def _add_datasets(self, user):
 
         #not able to add org to not existant org
         dataset = {'name': user + '_dataset_bad', 'owner_org': 'org_not_exist'}
-        self._call_api('package_create', dataset, user, 409)
+        self._call_api('package_create', dataset, user, 403)
 
     def test_07_add_datasets(self):
         self._add_datasets('org_admin')

From 3a60c2e2420350631742f8d0330be2f002d581eb Mon Sep 17 00:00:00 2001
From: Vitor Baptista <vitor@vitorbaptista.com>
Date: Wed, 16 Oct 2013 21:24:26 -0300
Subject: [PATCH 49/87] [#766] Stop Travis build if a dependency install fails

To do so, I needed to split our ./bin/travis-build in two files, one with the
deps install, and one with the test run. Doing so allows me to use the
"install" directive in Travis, and break the build as soon as something fails
with it.
---
 .travis.yml                                   |  3 ++-
 ...avis-build => travis-install-dependencies} | 26 +++----------------
 bin/travis-run-tests                          | 22 ++++++++++++++++
 3 files changed, 28 insertions(+), 23 deletions(-)
 rename bin/{travis-build => travis-install-dependencies} (73%)
 create mode 100755 bin/travis-run-tests

diff --git a/.travis.yml b/.travis.yml
index af450298b6c..43b6cc3776a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,7 +5,8 @@ python:
 env:
   - PGVERSION=9.1
   - PGVERSION=8.4
-script: ./bin/travis-build
+install: ./bin/travis-install-dependencies
+script: ./bin/travis-run-tests
 notifications:
   irc:
     channels:
diff --git a/bin/travis-build b/bin/travis-install-dependencies
similarity index 73%
rename from bin/travis-build
rename to bin/travis-install-dependencies
index 51e633781b4..306d6e1f166 100755
--- a/bin/travis-build
+++ b/bin/travis-install-dependencies
@@ -1,4 +1,7 @@
-#!/bin/sh
+#!/bin/bash
+
+# Exit immediately if any command fails
+set -e
 
 # Drop Travis' postgres cluster if we're building using a different pg version
 TRAVIS_PGVERSION='9.1'
@@ -51,24 +54,3 @@ else
 fi
 
 cat test-core.ini
-
-# Run mocha front-end tests
-# We need ckan to be running for some tests
-paster serve test-core.ini &
-sleep 5 # Make sure the server has fully started
-mocha-phantomjs http://localhost:5000/base/test/index.html
-# Did an error occur?
-MOCHA_ERROR=$?
-# We are done so kill ckan
-killall paster
-
-# And finally, run the nosetests
-nosetests --ckan --with-pylons=test-core.ini --nologcapture ckan ckanext
-# Did an error occur?
-NOSE_ERROR=$?
-
-[ "0" -ne "$MOCHA_ERROR" ] && echo MOCKA tests have failed
-[ "0" -ne "$NOSE_ERROR" ] && echo NOSE tests have failed
-
-# If an error occurred in our tests make sure travis knows
-exit `expr $MOCHA_ERROR + $NOSE_ERROR`
diff --git a/bin/travis-run-tests b/bin/travis-run-tests
new file mode 100755
index 00000000000..83f6d333e0d
--- /dev/null
+++ b/bin/travis-run-tests
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Run mocha front-end tests
+# We need ckan to be running for some tests
+paster serve test-core.ini &
+sleep 5 # Make sure the server has fully started
+mocha-phantomjs http://localhost:5000/base/test/index.html
+# Did an error occur?
+MOCHA_ERROR=$?
+# We are done so kill ckan
+killall paster
+
+# And finally, run the nosetests
+nosetests --ckan --with-pylons=test-core.ini --nologcapture ckan ckanext
+# Did an error occur?
+NOSE_ERROR=$?
+
+[ "0" -ne "$MOCHA_ERROR" ] && echo MOCKA tests have failed
+[ "0" -ne "$NOSE_ERROR" ] && echo NOSE tests have failed
+
+# If an error occurred in our tests make sure travis knows
+exit `expr $MOCHA_ERROR + $NOSE_ERROR`

From 1aee242081710ddb1283d3cb9b0987d209273b53 Mon Sep 17 00:00:00 2001
From: Vitor Baptista <vitor@vitorbaptista.com>
Date: Wed, 16 Oct 2013 22:04:02 -0300
Subject: [PATCH 50/87] [#766] Remove redundant datastore_test psql user
 creation

---
 bin/travis-install-dependencies | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/bin/travis-install-dependencies b/bin/travis-install-dependencies
index 306d6e1f166..dcbbf09b316 100755
--- a/bin/travis-install-dependencies
+++ b/bin/travis-install-dependencies
@@ -46,8 +46,7 @@ paster db init -c test-core.ini
 # If Postgres >= 9.0, we don't need to use datastore's legacy mode.
 if [ $PGVERSION != '8.4' ]
 then
-  psql -c 'CREATE USER datastore_default;' -U postgres
-  sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/datastore_default@\/datastore_test/' test-core.ini
+  sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/datastore_default:pass@\/datastore_test/' test-core.ini
   paster datastore set-permissions postgres -c test-core.ini
 else
   sed -i -e 's/.*datastore.read_url.*//' test-core.ini

From 5af301903da9b8642e7c8b775c227f6c39fe0945 Mon Sep 17 00:00:00 2001
From: John Martin <me@johnmart.in>
Date: Thu, 17 Oct 2013 10:32:42 +0100
Subject: [PATCH 51/87] [#1200] Tidies up the resource data view into something
 a little nicer

---
 ckan/public/base/javascript/main.js       |  1 +
 ckan/public/base/less/activity.less       | 18 ++++++
 ckan/templates/package/resource_data.html | 75 ++++++++++++++++-------
 3 files changed, 71 insertions(+), 23 deletions(-)

diff --git a/ckan/public/base/javascript/main.js b/ckan/public/base/javascript/main.js
index e307fe8403c..57331d782e6 100644
--- a/ckan/public/base/javascript/main.js
+++ b/ckan/public/base/javascript/main.js
@@ -36,6 +36,7 @@ this.ckan = this.ckan || {};
       ckan.i18n.load(data);
       ckan.module.initialize();
     });
+    $('[data-target="popover"').popover();
   };
 
   /* Returns a full url for the current site with the provided path appended.
diff --git a/ckan/public/base/less/activity.less b/ckan/public/base/less/activity.less
index dc75ede5e37..6071fabc5f0 100644
--- a/ckan/public/base/less/activity.less
+++ b/ckan/public/base/less/activity.less
@@ -56,6 +56,9 @@
 			.border-radius(100px);
 			.box-shadow(0 1px 2px rgba(0, 0, 0, 0.2));
 		}
+		&.no-avatar p {
+			margin-left: 40px;
+		}
 	}
 	.load-less {
 		margin-bottom: 15px;
@@ -76,11 +79,26 @@
 		float: right;
 		text-decoration: none;
 	}
+	.popover-content {
+		font-size: @baseFontSize;
+		line-height: @baseLineHeight;
+		color: @layoutTextColor;
+		word-break: break-all;
+		dl {
+			margin: 0;
+			dd {
+				margin-left: 0;
+				margin-bottom: 10px;
+			}
+		}
+	}
 }
 
 // colors
 .activity .item {
 	& .icon { background-color: @activityColorBlank; } // Non defined
+	&.failure .icon { background-color: @activityColorDelete; }
+	&.success .icon { background-color: @activityColorNew; }
 	&.added-tag .icon { background-color: spin(@activityColorNew, 60); }
 	&.changed-group .icon { background-color: @activityColorModify; }
 	&.changed-package .icon { background-color: spin(@activityColorModify, 20); }
diff --git a/ckan/templates/package/resource_data.html b/ckan/templates/package/resource_data.html
index 1179ed3bfec..4f19a6d11e6 100644
--- a/ckan/templates/package/resource_data.html
+++ b/ckan/templates/package/resource_data.html
@@ -5,35 +5,64 @@
 {% block primary_content_inner %}
 
   {% set action = h.url_for(controller='ckanext.datapusher.plugin:ResourceDataController', action='resource_data', id=pkg.name, resource_id=res.id) %}
+  {% set show_table = true %}
 
   <form method="post" action="{{ action }}" >
-
     <button class="btn btn-primary" name="save" type="submit">
-      {{ _('Upload Data') }}
+      <i class="icon-cloud-upload"></i> {{ _('Upload Data') }}
     </button>
+  </form>
 
-    <h3>
-      {{ _('Status:') }}
-      {{ status.status.capitalize() if status.status else _('Not Uploaded Yet') }}
-    </h3>
-
-    {% if status.error and status.error.message %}
-      <div class="alert alert-error">
-        <strong>{{ _('Upload Error') }}</strong> &mdash; {{ status.error.message }}
-      </div>
-    {% endif %}
-
-    {% if status.status %}
-      <h3>{{ _('Last Updated: ') }} {{ h.time_ago_from_timestamp(status.last_updated) }} </h3>
-    {% endif %}
+  {% if status.error and status.error.message %}
+    {% set show_table = false %}
+    <div class="alert alert-error">
+      <strong>{{ _('Upload error:') }}</strong> {{ status.error.message }}
+    </div>
+  {% elif status.task_info.error %}
+    {% set show_table = false %}
+    <div class="alert alert-error">
+      <strong>{{ _('Error:') }}</strong> {{ status.task_info.error }}
+    </div>
+  {% endif %}
 
-    {% if status.status and status.task_info %}
-      <h3>{{ _('Upload Log') }}</h3>
-      {{ h.debug_inspect(status.task_info.logs) }}
-    {% endif %} 
-    
-    {{ status }}
+  <table class="table table-bordered">
+    <colgroup>
+      <col width="150">
+      <col>
+    </colgroup>
+    <tr>
+      <th>{{ _('Status') }}</th>
+      <td>{{ status.status.capitalize() if status.status else _('Not Uploaded Yet') }}</td>
+    </tr>
+    <tr>
+      <th>{{ _('Last updated') }}</th>
+      <td>{{ h.time_ago_from_timestamp(status.last_updated) if status.status else _('Never') }}</td>
+    </tr>
+  </table>
 
-  </form>
+  {% if status.status and status.task_info and show_table %}
+    <h3>{{ _('Upload Log') }}</h3>
+    <ul class="activity">
+      {% for item in status.task_info.logs %}
+        {% set icon = 'ok' if item.level == 'INFO' else 'exclamation' %}
+        {% set class = ' failure' if icon == 'exclamation' else ' success' %}
+        {% set popover_content = 'test' %}
+        <li class="item no-avatar{{ class }}">
+          <i class="icon icon-{{ icon }}"></i>
+          <p>
+            {{ item.message | urlize }}<br>
+            <span class="date">
+              {{ h.time_ago_from_timestamp(item.timestamp) }}
+              <a href="#" data-target="popover" data-content="<dl>{% for key, value in item.iteritems() %}<dt>{{ key }}</dt><dd>{{ value }}</dd>{% endfor %}</dl>" data-html="true">more info</a>
+            </span>
+          </p>
+        </li>
+      {% endfor %}
+      <li class="item no-avatar">
+        <i class="icon icon-info"></i>
+        <p class="muted">{{ _('End of log') }}</p>
+      </li>
+    </ul>
+  {% endif %}
 
 {% endblock %}

From 6cc143f9deec75bf13fe2e517bead4ae90b91b50 Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Thu, 17 Oct 2013 13:47:19 +0100
Subject: [PATCH 52/87] [#1262] make tests pass

---
 ckan/config/middleware.py |  3 ++-
 ckan/lib/uploader.py      | 41 +++++++++++++++++++++++----------------
 2 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py
index 7699596138d..efde540054a 100644
--- a/ckan/config/middleware.py
+++ b/ckan/config/middleware.py
@@ -23,6 +23,7 @@
 from ckan.plugins import PluginImplementations
 from ckan.plugins.interfaces import IMiddleware
 from ckan.lib.i18n import get_locales_from_config
+import ckan.lib.uploader as uploader
 
 from ckan.config.environment import load_environment
 import ckan.lib.app_globals as app_globals
@@ -148,7 +149,7 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf):
                 cache_max_age=static_max_age)
         static_parsers = [static_app, app]
 
-        storage_directory = config.get('ckan.storage_path')
+        storage_directory = uploader.get_storage_path()
         if storage_directory:
             path = os.path.join(storage_directory, 'storage')
             try:
diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py
index 880488b8161..867e28cfcb0 100644
--- a/ckan/lib/uploader.py
+++ b/ckan/lib/uploader.py
@@ -10,6 +10,7 @@
 
 _storage_path = None
 
+
 def get_storage_path():
     '''Function to cache storage path'''
     global _storage_path
@@ -27,21 +28,24 @@ def get_storage_path():
             _storage_path = ofs_storage_dir
             return _storage_path
         elif ofs_impl:
-            log.critical('''We only support local file storage form version 2.2 of ckan
-                          please specify ckan.storage_path in your config for your uploads''')
+            log.critical('''We only support local file storage form version 2.2
+                         of ckan please specify ckan.storage_path in your
+                         config for your uploads''')
             _storage_path = False
         else:
-            log.critical('''Please specify a ckan.storage_path in your config for your uploads''')
+            log.critical('''Please specify a ckan.storage_path in your config
+                         for your uploads''')
             _storage_path = False
 
     return _storage_path
 
 
-
 class Upload(object):
     def __init__(self, object_type, old_filename=None):
-        ''' Setup upload by creating  a subdirectory of the storage directory of name
-        object_type. old_filename is the name of the file in the url field last time'''
+        ''' Setup upload by creating  a subdirectory of the storage directory
+        of name object_type. old_filename is the name of the file in the url
+        field last time'''
+
         self.storage_path = None
         self.filename = None
         self.filepath = None
@@ -60,20 +64,21 @@ def __init__(self, object_type, old_filename=None):
             self.old_filepath = os.path.join(self.storage_path, old_filename)
 
     def update_data_dict(self, data_dict, url_field, file_field, clear_field):
-        ''' Manipulate data from the data_dict.  url_field is the name of the field
-        where the upload is going to be. file_field is name of the key where the
-        FieldStorage is kept (i.e the field where the file data actually is). clear_field
-        is the name of a boolean field which requests the upload to be deleted.  This needs
-        to be called before it reaches any validators'''
-
-        if not self.storage_path:
-            return
+        ''' Manipulate data from the data_dict.  url_field is the name of the
+        field where the upload is going to be. file_field is name of the key
+        where the FieldStorage is kept (i.e the field where the file data
+        actually is). clear_field is the name of a boolean field which
+        requests the upload to be deleted.  This needs to be called before
+        it reaches any validators'''
 
         self.url = data_dict.get(url_field, '')
         self.clear = data_dict.pop(clear_field, None)
         self.file_field = file_field
         self.upload_field_storage = data_dict.pop(file_field, None)
 
+        if not self.storage_path:
+            return
+
         if isinstance(self.upload_field_storage, cgi.FieldStorage):
             self.filename = self.upload_field_storage.filename
             self.filename = str(datetime.datetime.utcnow()) + self.filename
@@ -90,9 +95,11 @@ def update_data_dict(self, data_dict, url_field, file_field, clear_field):
                 data_dict[url_field] = ''
 
     def upload(self, max_size=2):
-        ''' Actually upload the file.  This should happen just before a commit but after
-        the data has been validated and flushed to the db.  This is so we do not store anything
-        unless the request is actually good. Max_size is size in MB maximum of the file'''
+        ''' Actually upload the file.
+        This should happen just before a commit but after the data has
+        been validated and flushed to the db. This is so we do not store
+        anything unless the request is actually good.
+        max_size is size in MB maximum of the file'''
 
         if self.filename:
             output_file = open(self.tmp_filepath, 'wb')

From 7ab8bd9e9de8ff02a177db571e800d7383816c77 Mon Sep 17 00:00:00 2001
From: Samuele Santi <redshadow@hackzine.org>
Date: Fri, 18 Oct 2013 12:48:23 +0200
Subject: [PATCH 53/87] pep8

---
 setup.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index 3f5115bfa8f..04fc1647fec 100644
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,8 @@
     use_setuptools()
     from setuptools import setup, find_packages
 
-from ckan import __version__, __description__, __long_description__, __license__
+from ckan import (__version__, __description__, __long_description__,
+                  __license__)
 
 setup(
     name='ckan',
@@ -16,7 +17,7 @@
     url='http://ckan.org/',
     description=__description__,
     keywords='data packaging component tool server',
-    long_description =__long_description__,
+    long_description=__long_description__,
     zip_safe=False,
     packages=find_packages(exclude=['ez_setup']),
     namespace_packages=['ckanext', 'ckanext.stats'],
@@ -28,7 +29,7 @@
         'migration/tests/test_dumps/*',
         'migration/versions/*',
     ]},
-    message_extractors = {
+    message_extractors={
         'ckan': [
             ('**.py', 'python', None),
             ('**.js', 'javascript', None),

From 1ca945f79f600d6e4521e04cc5b90a75834dce4b Mon Sep 17 00:00:00 2001
From: Samuele Santi <redshadow@hackzine.org>
Date: Fri, 18 Oct 2013 12:57:29 +0200
Subject: [PATCH 54/87] Entry points converted to dict

---
 setup.py | 186 ++++++++++++++++++++++++++++---------------------------
 1 file changed, 94 insertions(+), 92 deletions(-)

diff --git a/setup.py b/setup.py
index 04fc1647fec..c40a746d72b 100644
--- a/setup.py
+++ b/setup.py
@@ -8,6 +8,99 @@
 from ckan import (__version__, __description__, __long_description__,
                   __license__)
 
+entry_points = {
+    'nose.plugins.0.10': [
+        'main = ckan.ckan_nose_plugin:CkanNose',
+    ],
+    'paste.app_factory': [
+        'main = ckan.config.middleware:make_app',
+    ],
+    'paste.app_install': [
+        'main = ckan.config.install:CKANInstaller',
+    ],
+    'paste.paster_command': [
+        'db = ckan.lib.cli:ManageDb',
+        'create-test-data = ckan.lib.cli:CreateTestDataCommand',
+        'sysadmin = ckan.lib.cli:Sysadmin',
+        'user = ckan.lib.cli:UserCmd',
+        'dataset = ckan.lib.cli:DatasetCmd',
+        'search-index = ckan.lib.cli:SearchIndexCommand',
+        'ratings = ckan.lib.cli:Ratings',
+        'notify = ckan.lib.cli:Notification',
+        'celeryd = ckan.lib.cli:Celery',
+        'rdf-export = ckan.lib.cli:RDFExport',
+        'tracking = ckan.lib.cli:Tracking',
+        'plugin-info = ckan.lib.cli:PluginInfo',
+        'profile = ckan.lib.cli:Profile',
+        'color = ckan.lib.cli:CreateColorSchemeCommand',
+        'check-po-files = ckan.i18n.check_po_files:CheckPoFiles',
+        'trans = ckan.lib.cli:TranslationsCommand',
+        'minify = ckan.lib.cli:MinifyCommand',
+        'less = ckan.lib.cli:LessCommand',
+        'datastore = ckanext.datastore.commands:SetupDatastoreCommand',
+        'front-end-build = ckan.lib.cli:FrontEndBuildCommand',
+    ],
+    'console_scripts': [
+        'ckan-admin = bin.ckan_admin:Command',
+    ],
+    'paste.paster_create_template': [
+        'ckanext = ckan.pastertemplates:CkanextTemplate',
+    ],
+    'ckan.forms': [
+        'standard = ckan.forms.package:get_standard_fieldset',
+        'package = ckan.forms.package:get_standard_fieldset',
+        'group = ckan.forms.group:get_group_fieldset',
+        'package_group = ckan.forms.group:get_package_group_fieldset',
+    ],
+    'ckan.search': [
+        'sql = ckan.lib.search.sql:SqlSearchBackend',
+        'solr = ckan.lib.search.solr_backend:SolrSearchBackend',
+    ],
+    'ckan.plugins': [
+        'synchronous_search = ckan.lib.search:SynchronousSearchPlugin',
+        'stats = ckanext.stats.plugin:StatsPlugin',
+        'publisher_form = ckanext.publisher_form.forms:PublisherForm',
+        'publisher_dataset_form = ckanext.publisher_form.forms:PublisherDatasetForm',
+        'multilingual_dataset = ckanext.multilingual.plugin:MultilingualDataset',
+        'multilingual_group = ckanext.multilingual.plugin:MultilingualGroup',
+        'multilingual_tag = ckanext.multilingual.plugin:MultilingualTag',
+        'organizations = ckanext.organizations.forms:OrganizationForm',
+        'organizations_dataset = ckanext.organizations.forms:OrganizationDatasetForm',
+        'datastore = ckanext.datastore.plugin:DatastorePlugin',
+        'test_tag_vocab_plugin = ckanext.test_tag_vocab_plugin:MockVocabTagsPlugin',
+        'resource_proxy = ckanext.resourceproxy.plugin:ResourceProxy',
+        'text_preview = ckanext.textpreview.plugin:TextPreview',
+        'pdf_preview = ckanext.pdfpreview.plugin:PdfPreview',
+        'recline_preview = ckanext.reclinepreview.plugin:ReclinePreview',
+        'example_itemplatehelpers = ckanext.example_itemplatehelpers.plugin:ExampleITemplateHelpersPlugin',
+        'example_idatasetform = ckanext.example_idatasetform.plugin:ExampleIDatasetFormPlugin',
+        'example_iauthfunctions_v1 = ckanext.example_iauthfunctions.plugin_v1:ExampleIAuthFunctionsPlugin',
+        'example_iauthfunctions_v2 = ckanext.example_iauthfunctions.plugin_v2:ExampleIAuthFunctionsPlugin',
+        'example_iauthfunctions_v3 = ckanext.example_iauthfunctions.plugin_v3:ExampleIAuthFunctionsPlugin',
+        'example_iauthfunctions = ckanext.example_iauthfunctions.plugin:ExampleIAuthFunctionsPlugin',
+    ],
+    'ckan.system_plugins': [
+        'domain_object_mods = ckan.model.modification:DomainObjectModificationExtension',
+    ],
+    'ckan.test_plugins': [
+        'routes_plugin = tests.ckantestplugins:RoutesPlugin',
+        'mapper_plugin = tests.ckantestplugins:MapperPlugin',
+        'session_plugin = tests.ckantestplugins:SessionPlugin',
+        'mapper_plugin2 = tests.ckantestplugins:MapperPlugin2',
+        'authorizer_plugin = tests.ckantestplugins:AuthorizerPlugin',
+        'test_observer_plugin = tests.ckantestplugins:PluginObserverPlugin',
+        'action_plugin = tests.ckantestplugins:ActionPlugin',
+        'auth_plugin = tests.ckantestplugins:AuthPlugin',
+        'test_group_plugin = tests.ckantestplugins:MockGroupControllerPlugin',
+        'test_package_controller_plugin = tests.ckantestplugins:MockPackageControllerPlugin',
+        'test_resource_preview = tests.ckantestplugins:MockResourcePreviewExtension',
+        'test_json_resource_preview = tests.ckantestplugins:JsonMockResourcePreviewExtension',
+    ],
+    'babel.extractors': [
+        'ckan = ckan.lib.extract:extract_ckan',
+    ],
+}
+
 setup(
     name='ckan',
     version=__version__,
@@ -56,98 +149,7 @@
             }),
         ]
     },
-    entry_points="""
-    [nose.plugins.0.10]
-    main = ckan.ckan_nose_plugin:CkanNose
-
-    [paste.app_factory]
-    main = ckan.config.middleware:make_app
-
-    [paste.app_install]
-    main = ckan.config.install:CKANInstaller
-
-    [paste.paster_command]
-    db = ckan.lib.cli:ManageDb
-    create-test-data = ckan.lib.cli:CreateTestDataCommand
-    sysadmin = ckan.lib.cli:Sysadmin
-    user = ckan.lib.cli:UserCmd
-    dataset = ckan.lib.cli:DatasetCmd
-    search-index = ckan.lib.cli:SearchIndexCommand
-    ratings = ckan.lib.cli:Ratings
-    notify = ckan.lib.cli:Notification
-    celeryd = ckan.lib.cli:Celery
-    rdf-export = ckan.lib.cli:RDFExport
-    tracking = ckan.lib.cli:Tracking
-    plugin-info = ckan.lib.cli:PluginInfo
-    profile = ckan.lib.cli:Profile
-    color = ckan.lib.cli:CreateColorSchemeCommand
-    check-po-files = ckan.i18n.check_po_files:CheckPoFiles
-    trans = ckan.lib.cli:TranslationsCommand
-    minify = ckan.lib.cli:MinifyCommand
-    less = ckan.lib.cli:LessCommand
-    datastore = ckanext.datastore.commands:SetupDatastoreCommand
-    front-end-build = ckan.lib.cli:FrontEndBuildCommand
-
-
-    [console_scripts]
-    ckan-admin = bin.ckan_admin:Command
-
-    [paste.paster_create_template]
-    ckanext=ckan.pastertemplates:CkanextTemplate
-
-    [ckan.forms]
-    standard = ckan.forms.package:get_standard_fieldset
-    package = ckan.forms.package:get_standard_fieldset
-    group = ckan.forms.group:get_group_fieldset
-    package_group = ckan.forms.group:get_package_group_fieldset
-
-    [ckan.search]
-    sql = ckan.lib.search.sql:SqlSearchBackend
-    solr = ckan.lib.search.solr_backend:SolrSearchBackend
-
-    [ckan.plugins]
-    synchronous_search = ckan.lib.search:SynchronousSearchPlugin
-    stats=ckanext.stats.plugin:StatsPlugin
-    publisher_form=ckanext.publisher_form.forms:PublisherForm
-    publisher_dataset_form=ckanext.publisher_form.forms:PublisherDatasetForm
-    multilingual_dataset=ckanext.multilingual.plugin:MultilingualDataset
-    multilingual_group=ckanext.multilingual.plugin:MultilingualGroup
-    multilingual_tag=ckanext.multilingual.plugin:MultilingualTag
-    organizations=ckanext.organizations.forms:OrganizationForm
-    organizations_dataset=ckanext.organizations.forms:OrganizationDatasetForm
-    datastore=ckanext.datastore.plugin:DatastorePlugin
-    test_tag_vocab_plugin=ckanext.test_tag_vocab_plugin:MockVocabTagsPlugin
-    resource_proxy=ckanext.resourceproxy.plugin:ResourceProxy
-    text_preview=ckanext.textpreview.plugin:TextPreview
-    pdf_preview=ckanext.pdfpreview.plugin:PdfPreview
-    recline_preview=ckanext.reclinepreview.plugin:ReclinePreview
-    example_itemplatehelpers=ckanext.example_itemplatehelpers.plugin:ExampleITemplateHelpersPlugin
-    example_idatasetform=ckanext.example_idatasetform.plugin:ExampleIDatasetFormPlugin
-    example_iauthfunctions_v1=ckanext.example_iauthfunctions.plugin_v1:ExampleIAuthFunctionsPlugin
-    example_iauthfunctions_v2=ckanext.example_iauthfunctions.plugin_v2:ExampleIAuthFunctionsPlugin
-    example_iauthfunctions_v3=ckanext.example_iauthfunctions.plugin_v3:ExampleIAuthFunctionsPlugin
-    example_iauthfunctions=ckanext.example_iauthfunctions.plugin:ExampleIAuthFunctionsPlugin
-
-    [ckan.system_plugins]
-    domain_object_mods = ckan.model.modification:DomainObjectModificationExtension
-
-    [ckan.test_plugins]
-    routes_plugin=tests.ckantestplugins:RoutesPlugin
-    mapper_plugin=tests.ckantestplugins:MapperPlugin
-    session_plugin=tests.ckantestplugins:SessionPlugin
-    mapper_plugin2=tests.ckantestplugins:MapperPlugin2
-    authorizer_plugin=tests.ckantestplugins:AuthorizerPlugin
-    test_observer_plugin=tests.ckantestplugins:PluginObserverPlugin
-    action_plugin=tests.ckantestplugins:ActionPlugin
-    auth_plugin=tests.ckantestplugins:AuthPlugin
-    test_group_plugin=tests.ckantestplugins:MockGroupControllerPlugin
-    test_package_controller_plugin=tests.ckantestplugins:MockPackageControllerPlugin
-    test_resource_preview=tests.ckantestplugins:MockResourcePreviewExtension
-    test_json_resource_preview=tests.ckantestplugins:JsonMockResourcePreviewExtension
-
-    [babel.extractors]
-    ckan = ckan.lib.extract:extract_ckan
-    """,
+    entry_points=entry_points,
     # setup.py test command needs a TestSuite so does not work with py.test
     # test_suite = 'nose.collector',
     # tests_require=[ 'py >= 0.8.0-alpha2' ]

From 4f6e390e03759aa0333e534ef7e9f521725ec8d9 Mon Sep 17 00:00:00 2001
From: Nigel Babu <nigelbabu@gmail.com>
Date: Mon, 21 Oct 2013 13:06:41 +0530
Subject: [PATCH 55/87] Create new schema for related update

---
 ckan/logic/action/update.py | 2 +-
 ckan/logic/schema.py        | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py
index 3db2d0d19cf..13fc147b50b 100644
--- a/ckan/logic/action/update.py
+++ b/ckan/logic/action/update.py
@@ -132,7 +132,7 @@ def related_update(context, data_dict):
     id = _get_or_bust(data_dict, "id")
 
     session = context['session']
-    schema = context.get('schema') or schema_.default_related_schema()
+    schema = context.get('schema') or schema_.default_update_related_schema()
 
     related = model.Related.get(id)
     context["related"] = related
diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py
index 41e3c724656..9fb859fde65 100644
--- a/ckan/logic/schema.py
+++ b/ckan/logic/schema.py
@@ -331,6 +331,15 @@ def default_related_schema():
     return schema
 
 
+def default_update_related_schema():
+    schema = default_related_schema()
+    schema['id'] = [not_empty, unicode]
+    schema['title'] = [ignore_missing, unicode]
+    schema['type'] = [ignore_missing, unicode]
+    schema['owner_id'] = [ignore_missing, unicode]
+    return schema
+
+
 def default_extras_schema():
 
     schema = {

From 839a0981d2c92e99b3c08707da147d3116008b47 Mon Sep 17 00:00:00 2001
From: Nigel Babu <nigelbabu@gmail.com>
Date: Mon, 21 Oct 2013 15:58:06 +0530
Subject: [PATCH 56/87] Add tests for related actions

---
 ckan/tests/logic/test_action.py | 61 +++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py
index ed1907ebdb8..fac25a7ba86 100644
--- a/ckan/tests/logic/test_action.py
+++ b/ckan/tests/logic/test_action.py
@@ -1978,3 +1978,64 @@ def _assert_we_can_add_user_to_group(self, user_id, group_id):
         group_ids = [g.id for g in groups]
         assert res['success'] is True, res
         assert group.id in group_ids, (group, user_groups)
+
+
+class TestRelatedAction(WsgiAppCase):
+
+    sysadmin_user = None
+
+    normal_user = None
+
+    @classmethod
+    def setup_class(cls):
+        search.clear()
+        CreateTestData.create()
+        cls.sysadmin_user = model.User.get('testsysadmin')
+
+    @classmethod
+    def teardown_class(cls):
+        model.repo.rebuild_db()
+
+    def _add_basic_package(self, package_name=u'test_package', **kwargs):
+        package = {
+            'name': package_name,
+            'title': u'A Novel By Tolstoy',
+            'resources': [{
+                'description': u'Full text.',
+                'format': u'plain text',
+                'url': u'http://www.annakarenina.com/download/'
+            }]
+        }
+        package.update(kwargs)
+
+        postparams = '%s=1' % json.dumps(package)
+        res = self.app.post('/api/action/package_create', params=postparams,
+                            extra_environ={'Authorization': 'tester'})
+        return json.loads(res.body)['result']
+
+    def test_01_update_add_related_item(self):
+        package = self._add_basic_package()
+        related_item = {
+            "description": "Testing a Description",
+            "url": "http://example.com/image.png",
+            "title": "Testing",
+            "featured": 0,
+            "image_url": "http://example.com/image.png",
+            "type": "idea",
+            "dataset_id": package['id'],
+        }
+        related_item_json = json.dumps(related_item)
+        res_create = self.app.post('/api/action/related_create', params=related_item_json,
+                            extra_environ={'Authorization': 'tester'})
+        assert res_create.json['success'] == True
+
+        related_update = res_create.json['result']
+        related_update = {'id': related_update['id'], 'title': 'Updated'}
+        related_update_json = json.dumps(related_update)
+        res_update = self.app.post('/api/action/related_update', params=related_update_json,
+                            extra_environ={'Authorization': 'tester'})
+        assert res_update.json['success'] == True
+        res_update_json = res_update.json['result']
+        assert res_update_json['title'] == related_update['title']
+
+

From 5cc76acca3802922d37db5f341fe72bd4365c60c Mon Sep 17 00:00:00 2001
From: Stefan Oderbolz <stefan.oderbolz@liip.ch>
Date: Mon, 21 Oct 2013 15:26:25 +0200
Subject: [PATCH 57/87] Add organizations to the default facet titles

Currently this is only set in the package controller, so on all other pages that display the facets the title is not displayed correctly
---
 ckan/controllers/group.py | 3 ++-
 ckan/controllers/home.py  | 1 +
 ckan/lib/helpers.py       | 3 ++-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py
index 66bd533fd43..06d34496f2e 100644
--- a/ckan/controllers/group.py
+++ b/ckan/controllers/group.py
@@ -278,7 +278,8 @@ def pager_url(q=None, page=None):
 
             facets = OrderedDict()
 
-            default_facet_titles = {'groups': _('Groups'),
+            default_facet_titles = {'organization': _('Organizations'),
+                                    'groups': _('Groups'),
                                     'tags': _('Tags'),
                                     'res_format': _('Formats'),
                                     'license_id': _('License')}
diff --git a/ckan/controllers/home.py b/ckan/controllers/home.py
index da3d4b3901b..7aa15be87bd 100644
--- a/ckan/controllers/home.py
+++ b/ckan/controllers/home.py
@@ -67,6 +67,7 @@ def index(self):
             c.search_facets = query['search_facets']
 
             c.facet_titles = {
+                'organization': _('Organizations'),
                 'groups': _('Groups'),
                 'tags': _('Tags'),
                 'res_format': _('Formats'),
diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
index edd9746f0a8..9a2b83ddfb1 100644
--- a/ckan/lib/helpers.py
+++ b/ckan/lib/helpers.py
@@ -597,7 +597,8 @@ def get_facet_title(name):
     if config_title:
         return config_title
 
-    facet_titles = {'groups': _('Groups'),
+    facet_titles = {'organization': _('Organizations'),
+                    'groups': _('Groups'),
                     'tags': _('Tags'),
                     'res_format': _('Formats'),
                     'license': _('License'), }

From b355fc79b0aa6ca62e6b104095e5b9696424cac6 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Mon, 21 Oct 2013 09:32:31 -0700
Subject: [PATCH 58/87] Coveralls badge should show coverage default branch in
 readme

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 2eafd662e07..a85163653d8 100644
--- a/README.rst
+++ b/README.rst
@@ -6,7 +6,7 @@ CKAN: The Open Source Data Portal Software
     :alt: Build Status
 
 .. image:: https://coveralls.io/repos/okfn/ckan/badge.png?branch=coveralls
-	:target: https://coveralls.io/r/okfn/ckan?branch=coveralls
+	:target: https://coveralls.io/r/okfn/ckan
 	:alt: Test coverage
 
 **CKAN is the world’s leading open-source data portal platform**.

From 48f5eeb6cda7955bc89aac50ad27969bd5abfe04 Mon Sep 17 00:00:00 2001
From: Ian Ward <ian@excess.org>
Date: Mon, 21 Oct 2013 14:57:23 -0400
Subject: [PATCH 59/87] [#1281] DefaultOrganizationForm instead of hard-coded
 values in controller

---
 ckan/controllers/organization.py | 41 --------------------------------
 ckan/lib/plugins.py              | 39 +++++++++++++++++++++++++++++-
 2 files changed, 38 insertions(+), 42 deletions(-)

diff --git a/ckan/controllers/organization.py b/ckan/controllers/organization.py
index b44d95e057f..3907143a187 100644
--- a/ckan/controllers/organization.py
+++ b/ckan/controllers/organization.py
@@ -15,46 +15,5 @@ class OrganizationController(group.GroupController):
     # this makes us use organization actions
     group_type = 'organization'
 
-    def _group_form(self, group_type=None):
-        return 'organization/new_organization_form.html'
-
-    def _form_to_db_schema(self, group_type=None):
-        return group.lookup_group_plugin(group_type).form_to_db_schema()
-
-    def _db_to_form_schema(self, group_type=None):
-        '''This is an interface to manipulate data from the database
-        into a format suitable for the form (optional)'''
-        pass
-
-    def _setup_template_variables(self, context, data_dict, group_type=None):
-        pass
-
-    def _new_template(self, group_type):
-        return 'organization/new.html'
-
-    def _about_template(self, group_type):
-        return 'organization/about.html'
-
-    def _index_template(self, group_type):
-        return 'organization/index.html'
-
-    def _admins_template(self, group_type):
-        return 'organization/admins.html'
-
-    def _bulk_process_template(self, group_type):
-        return 'organization/bulk_process.html'
-
-    def _read_template(self, group_type):
-        return 'organization/read.html'
-
-    def _history_template(self, group_type):
-        return group.lookup_group_plugin(group_type).history_template()
-
-    def _edit_template(self, group_type):
-        return 'organization/edit.html'
-
-    def _activity_template(self, group_type):
-        return 'organization/activity_stream.html'
-
     def _guess_group_type(self, expecting_name=False):
         return 'organization'
diff --git a/ckan/lib/plugins.py b/ckan/lib/plugins.py
index 97263dcf7f8..40a398bd3f4 100644
--- a/ckan/lib/plugins.py
+++ b/ckan/lib/plugins.py
@@ -53,7 +53,8 @@ def lookup_group_plugin(group_type=None):
     """
     if group_type is None:
         return _default_group_plugin
-    return _group_plugins.get(group_type, _default_group_plugin)
+    return _group_plugins.get(group_type, _default_organization_plugin
+        if group_type == 'organization' else _default_group_plugin)
 
 
 def register_package_plugins(map):
@@ -418,3 +419,39 @@ def setup_template_variables(self, context, data_dict):
                 c.auth_for_change_state = True
             except logic.NotAuthorized:
                 c.auth_for_change_state = False
+
+
+class DefaultOrganizationForm(DefaultGroupForm):
+    def group_form(self):
+        return 'organization/new_organization_form.html'
+
+    def setup_template_variables(self, context, data_dict):
+        pass
+
+    def new_template(self):
+        return 'organization/new.html'
+
+    def about_template(self):
+        return 'organization/about.html'
+
+    def index_template(self):
+        return 'organization/index.html'
+
+    def admins_template(self):
+        return 'organization/admins.html'
+
+    def bulk_process_template(self):
+        return 'organization/bulk_process.html'
+
+    def read_template(self):
+        return 'organization/read.html'
+
+    # don't override history_template - use group template for history
+
+    def edit_template(self):
+        return 'organization/edit.html'
+
+    def activity_template(self):
+        return 'organization/activity_stream.html'
+
+_default_organization_plugin = DefaultOrganizationForm()

From 8afb07cec6b4e2d55bdb08de4f421dd20ab13a68 Mon Sep 17 00:00:00 2001
From: Vitor Baptista <vitor@vitorbaptista.com>
Date: Mon, 21 Oct 2013 18:15:56 -0300
Subject: [PATCH 60/87] Forgot to change branch specification in Coveralls'
 badge

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index a85163653d8..9c810209231 100644
--- a/README.rst
+++ b/README.rst
@@ -5,7 +5,7 @@ CKAN: The Open Source Data Portal Software
     :target: http://travis-ci.org/okfn/ckan
     :alt: Build Status
 
-.. image:: https://coveralls.io/repos/okfn/ckan/badge.png?branch=coveralls
+.. image:: https://coveralls.io/repos/okfn/ckan/badge.png
 	:target: https://coveralls.io/r/okfn/ckan
 	:alt: Test coverage
 

From ad4e7934d686205d760073826feb6925db3c0c5f Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Tue, 22 Oct 2013 18:47:16 +0100
Subject: [PATCH 61/87] [#1200] fixes from pull request

---
 ckan/config/routing.py             |  2 --
 ckan/model/meta.py                 |  2 +-
 ckanext/datapusher/logic/action.py | 15 ++++++++++-----
 ckanext/datapusher/plugin.py       | 15 +++++----------
 ckanext/datastore/helpers.py       | 15 ---------------
 5 files changed, 16 insertions(+), 33 deletions(-)
 delete mode 100644 ckanext/datastore/helpers.py

diff --git a/ckan/config/routing.py b/ckan/config/routing.py
index 66df3454b3d..016c65ae0e4 100644
--- a/ckan/config/routing.py
+++ b/ckan/config/routing.py
@@ -235,8 +235,6 @@ def make_map():
                   ])))
         m.connect('dataset_edit', '/dataset/edit/{id}', action='edit',
                   ckan_icon='edit')
-        m.connect('dataset_resources', '/dataset/resources/{id}',
-                  action='resources', ckan_icon='time')
         m.connect('dataset_followers', '/dataset/followers/{id}',
                   action='followers', ckan_icon='group')
         m.connect('dataset_activity', '/dataset/activity/{id}',
diff --git a/ckan/model/meta.py b/ckan/model/meta.py
index 0103019bb3a..2b6cb4581ba 100644
--- a/ckan/model/meta.py
+++ b/ckan/model/meta.py
@@ -162,5 +162,5 @@ def engine_is_sqlite(sa_engine=None):
 def engine_is_pg(sa_engine=None):
     # Returns true iff the engine is connected to a postgresql database.
     # According to http://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql
-    # all Postgres driver names start with `postgresql`
+    # all Postgres driver names start with `postgres`
     return (sa_engine or engine).url.drivername.startswith('postgres')
diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py
index cfbd4620d09..fb91a374243 100644
--- a/ckanext/datapusher/logic/action.py
+++ b/ckanext/datapusher/logic/action.py
@@ -131,13 +131,18 @@ def datapusher_hook(context, data_dict):
     ''' Update datapusher task. This action is typically called by the
     datapusher whenever the status of a job changes.
 
-    Expects a job with ``status`` and ``metadata`` with a ``resource_id``.
+    :param metadata: metadata produced by datapuser service must have
+       resource_id property.
+    :type metadata: dict
+    :param status: status of the job from the datapusher service
+    :type status: string
     '''
 
-    # TODO: use a schema to validate
+    metadata, status = _get_or_bust(data_dict, ['metadata', 'status'])
+
     p.toolkit.check_access('datapusher_submit', context, data_dict)
 
-    res_id = data_dict['metadata']['resource_id']
+    res_id = metadata.get('resource_id')
 
     task = p.toolkit.get_action('task_status_show')(context, {
         'entity_id': res_id,
@@ -145,7 +150,7 @@ def datapusher_hook(context, data_dict):
         'key': 'datapusher'
     })
 
-    task['state'] = data_dict['status']
+    task['state'] = status
     task['last_updated'] = str(datetime.datetime.now())
 
     p.toolkit.get_action('task_status_update')(context, task)
@@ -174,7 +179,7 @@ def datapusher_status(context, data_dict):
     datapusher_url = pylons.config.get('ckan.datapusher.url')
     if not datapusher_url:
         raise p.toolkit.ValidationError(
-            {'configuration': ['DataPusher not configured.']})
+            {'configuration': ['ckan.datapusher.url not in config file']})
 
     value = json.loads(task['value'])
     job_key = value.get('job_key')
diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py
index e6fa45fa61f..c53dd14790d 100644
--- a/ckanext/datapusher/plugin.py
+++ b/ckanext/datapusher/plugin.py
@@ -8,7 +8,7 @@
 import ckanext.datapusher.helpers as helpers
 import ckan.logic as logic
 import ckan.model as model
-from ckan.common import c, _, request
+import ckan.plugins.toolkit as toolkit
 
 log = logging.getLogger(__name__)
 _get_or_bust = logic.get_or_bust
@@ -24,10 +24,9 @@ class ResourceDataController(base.BaseController):
 
     def resource_data(self, id, resource_id):
 
-        if request.method == 'POST':
-            result = request.POST
+        if toolkit.request.method == 'POST':
             try:
-                c.pkg_dict = p.toolkit.get_action('datapusher_submit')(
+                toolkit.c.pkg_dict = p.toolkit.get_action('datapusher_submit')(
                     None, {'resource_id': resource_id}
                 )
             except logic.ValidationError:
@@ -41,10 +40,10 @@ def resource_data(self, id, resource_id):
             )
 
         try:
-            c.pkg_dict = p.toolkit.get_action('package_show')(
+            toolkit.c.pkg_dict = p.toolkit.get_action('package_show')(
                 None, {'id': id}
             )
-            c.resource = p.toolkit.get_action('resource_show')(
+            toolkit.c.resource = p.toolkit.get_action('resource_show')(
                 None, {'id': resource_id}
             )
         except logic.NotFound:
@@ -86,10 +85,6 @@ def configure(self, config):
             raise Exception(
                 'Config option `ckan.datapusher.url` has to be set.')
 
-        if not datapusher_url:
-            raise Exception(
-                'Config option `ckan.datapusher.secret_key` has to be set.')
-
     def notify(self, entity, operation=None):
         if isinstance(entity, model.Resource):
             if (operation == model.domain_object.DomainObjectOperation.new
diff --git a/ckanext/datastore/helpers.py b/ckanext/datastore/helpers.py
deleted file mode 100644
index bf06b2e063a..00000000000
--- a/ckanext/datastore/helpers.py
+++ /dev/null
@@ -1,15 +0,0 @@
-import ckan.plugins.toolkit as toolkit
-
-
-def datapusher_status(resource_id):
-    try:
-        return toolkit.get_action('datapusher_status')(
-            {}, {'resource_id': resource_id})
-    except toolkit.ObjectNotFound:
-        return {
-            'status': 'unknown'
-        }
-
-
-def datapusher_enabled():
-    return toolkit.get_action('datapusher_enabled')({})

From 4c46078e6737062d9b48c30e82afb5203f02c770 Mon Sep 17 00:00:00 2001
From: Vitor Baptista <vitor@vitorbaptista.com>
Date: Tue, 22 Oct 2013 17:25:43 -0300
Subject: [PATCH 62/87] [#766] Move solr configuration to travis-run-tests

I was unable to run Jetty-Solr's configuration on travis-install-dependencies.
Probably something to do with Bash's "set -e", but I was unable to pinpoint
exactly, so I left it into travis-run-tests. No big deal.
---
 bin/travis-install-dependencies | 6 ------
 bin/travis-run-tests            | 6 ++++++
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/bin/travis-install-dependencies b/bin/travis-install-dependencies
index dcbbf09b316..50b5ebd22cd 100755
--- a/bin/travis-install-dependencies
+++ b/bin/travis-install-dependencies
@@ -35,12 +35,6 @@ python setup.py develop
 # Install npm dpes for mocha
 npm install -g mocha-phantomjs phantomjs
 
-# Configure Solr
-echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
-# FIXME the solr schema cannot be hardcoded as it is dependent on the ckan version
-sudo cp ckan/config/solr/schema-2.0.xml /etc/solr/conf/schema.xml
-sudo service jetty restart
-
 paster db init -c test-core.ini
 
 # If Postgres >= 9.0, we don't need to use datastore's legacy mode.
diff --git a/bin/travis-run-tests b/bin/travis-run-tests
index 83f6d333e0d..f9a44de6f9e 100755
--- a/bin/travis-run-tests
+++ b/bin/travis-run-tests
@@ -1,5 +1,11 @@
 #!/bin/bash
 
+# Configure Solr
+echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
+# FIXME the solr schema cannot be hardcoded as it is dependent on the ckan version
+sudo cp ckan/config/solr/schema-2.0.xml /etc/solr/conf/schema.xml
+sudo service jetty restart
+
 # Run mocha front-end tests
 # We need ckan to be running for some tests
 paster serve test-core.ini &

From 852a23cda23a81ddc2374d53f67556d2a700ca7e Mon Sep 17 00:00:00 2001
From: Vitor Baptista <vitor@vitorbaptista.com>
Date: Tue, 22 Oct 2013 19:02:53 -0300
Subject: [PATCH 63/87] [#766] Use /bin/sh instead of /bin/bash

---
 bin/travis-run-tests | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/travis-run-tests b/bin/travis-run-tests
index f9a44de6f9e..760be3f458f 100755
--- a/bin/travis-run-tests
+++ b/bin/travis-run-tests
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 
 # Configure Solr
 echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty

From d30986d9f585e5897992ed97146e04672c14d54e Mon Sep 17 00:00:00 2001
From: Stefan Oderbolz <stefan.oderbolz@liip.ch>
Date: Wed, 9 Oct 2013 18:48:12 +0200
Subject: [PATCH 64/87] Added IOrganizationController to MultilingualGroup

This ensures that organizations are being translated just like groups
---
 ckanext/multilingual/plugin.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ckanext/multilingual/plugin.py b/ckanext/multilingual/plugin.py
index a554042f209..a4ee31e3b48 100644
--- a/ckanext/multilingual/plugin.py
+++ b/ckanext/multilingual/plugin.py
@@ -1,7 +1,7 @@
 import sets
 import ckan
 from ckan.plugins import SingletonPlugin, implements, IPackageController
-from ckan.plugins import IGroupController, ITagController
+from ckan.plugins import IGroupController, IOrganizationController, ITagController
 import pylons
 import ckan.logic.action.get as action_get
 from pylons import config
@@ -253,6 +253,7 @@ class MultilingualGroup(SingletonPlugin):
 
     '''
     implements(IGroupController, inherit=True)
+    implements(IOrganizationController, inherit=True)
 
     def before_view(self, data_dict):
         translated_data_dict = translate_data_dict(data_dict)

From 904dbf045f47f26692d467edca2ab1a6133e5e60 Mon Sep 17 00:00:00 2001
From: John Glover <j@johnglover.net>
Date: Wed, 23 Oct 2013 15:24:45 +0200
Subject: [PATCH 65/87] [#1274] PEP8

---
 ckan/tests/test_coding_standards.py           |   1 -
 .../tests/test_multilingual_plugin.py         | 271 +++++++++---------
 2 files changed, 134 insertions(+), 138 deletions(-)

diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py
index f0b7469c3f4..349f4785f6a 100644
--- a/ckan/tests/test_coding_standards.py
+++ b/ckan/tests/test_coding_standards.py
@@ -828,7 +828,6 @@ class TestPep8(object):
         'ckanext/example_idatasetform/plugin.py',
         'ckanext/example_itemplatehelpers/plugin.py',
         'ckanext/multilingual/plugin.py',
-        'ckanext/multilingual/tests/test_multilingual_plugin.py',
         'ckanext/reclinepreview/plugin.py',
         'ckanext/reclinepreview/tests/test_preview.py',
         'ckanext/resourceproxy/plugin.py',
diff --git a/ckanext/multilingual/tests/test_multilingual_plugin.py b/ckanext/multilingual/tests/test_multilingual_plugin.py
index ecc22b745f7..d7c4ec8a7f0 100644
--- a/ckanext/multilingual/tests/test_multilingual_plugin.py
+++ b/ckanext/multilingual/tests/test_multilingual_plugin.py
@@ -9,10 +9,11 @@
 import paste.fixture
 import pylons.test
 
-class TestDatasetTermTranslation(ckan.tests.html_check.HtmlCheckMethods):
-    '''Test the translation of datasets by the multilingual_dataset plugin.
+_create_test_data = ckan.lib.create_test_data
+
 
-    '''
+class TestDatasetTermTranslation(ckan.tests.html_check.HtmlCheckMethods):
+    'Test the translation of datasets by the multilingual_dataset plugin.'
     @classmethod
     def setup(cls):
         cls.app = paste.fixture.TestApp(pylons.test.pylonsapp)
@@ -20,24 +21,20 @@ def setup(cls):
         ckan.plugins.load('multilingual_group')
         ckan.plugins.load('multilingual_tag')
         ckan.tests.setup_test_search_index()
-        ckan.lib.create_test_data.CreateTestData.create_translations_test_data()
+        _create_test_data.CreateTestData.create_translations_test_data()
         # Add translation terms that match a couple of group names and package
         # names. Group names and package names should _not_ get translated even
         # if there are terms matching them, because they are used to form URLs.
         for term in ('roger', 'david', 'annakarenina', 'warandpeace'):
             for lang_code in ('en', 'de', 'fr'):
-                data_dict = {
-                        'term': term,
-                        'term_translation': 'this should not be rendered',
-                        'lang_code': lang_code,
-                        }
-                context = {
-                    'model': ckan.model,
-                    'session': ckan.model.Session,
-                    'user': 'testsysadmin',
-                }
-                ckan.logic.action.update.term_translation_update(context,
-                        data_dict)
+                data_dict = {'term': term,
+                             'term_translation': 'this should not be rendered',
+                             'lang_code': lang_code}
+                context = {'model': ckan.model,
+                           'session': ckan.model.Session,
+                           'user': 'testsysadmin'}
+                ckan.logic.action.update.term_translation_update(
+                    context, data_dict)
 
     @classmethod
     def teardown(cls):
@@ -54,34 +51,34 @@ def test_dataset_read_translation(self):
         '''
         # Fetch the dataset view page for a number of different languages and
         # test for the presence of translated and not translated terms.
-        offset = routes.url_for(controller='package', action='read',
-                id='annakarenina')
+        offset = routes.url_for(
+            controller='package', action='read', id='annakarenina')
         for (lang_code, translations) in (
-                ('de', ckan.lib.create_test_data.german_translations),
-                ('fr', ckan.lib.create_test_data.french_translations),
-                ('en', ckan.lib.create_test_data.english_translations),
+                ('de', _create_test_data.german_translations),
+                ('fr', _create_test_data.french_translations),
+                ('en', _create_test_data.english_translations),
                 ('pl', {})):
             response = self.app.get(offset, status=200,
-                    extra_environ={'CKAN_LANG': lang_code,
-                        'CKAN_CURRENT_URL': offset})
+                                    extra_environ={'CKAN_LANG': lang_code,
+                                                   'CKAN_CURRENT_URL': offset})
             terms = ('A Novel By Tolstoy',
-                'Index of the novel',
-                'russian',
-                'tolstoy',
-                "Dave's books",
-                "Roger's books",
-                'romantic novel',
-                'book',
-                '123',
-                '456',
-                '789',
-                'plain text',
-            )
+                     'Index of the novel',
+                     'russian',
+                     'tolstoy',
+                     "Dave's books",
+                     "Roger's books",
+                     'romantic novel',
+                     'book',
+                     '123',
+                     '456',
+                     '789',
+                     'plain text',)
             for term in terms:
                 if term in translations:
                     assert translations[term] in response
-                elif term in ckan.lib.create_test_data.english_translations:
-                    assert ckan.lib.create_test_data.english_translations[term] in response
+                elif term in _create_test_data.english_translations:
+                    assert (_create_test_data.english_translations[term]
+                            in response)
                 else:
                     assert term in response
             for tag_name in ('123', '456', '789', 'russian', 'tolstoy'):
@@ -96,23 +93,25 @@ def test_tag_read_translation(self):
 
         '''
         for tag_name in ('123', '456', '789', 'russian', 'tolstoy'):
-            offset = routes.url_for(controller='tag', action='read',
-                    id=tag_name)
+            offset = routes.url_for(
+                controller='tag', action='read', id=tag_name)
             for (lang_code, translations) in (
-                    ('de', ckan.lib.create_test_data.german_translations),
-                    ('fr', ckan.lib.create_test_data.french_translations),
-                    ('en', ckan.lib.create_test_data.english_translations),
+                    ('de', _create_test_data.german_translations),
+                    ('fr', _create_test_data.french_translations),
+                    ('en', _create_test_data.english_translations),
                     ('pl', {})):
-                response = self.app.get(offset, status=200,
-                        extra_environ={'CKAN_LANG': lang_code,
-                            'CKAN_CURRENT_URL': offset})
+                response = self.app.get(
+                    offset,
+                    status=200,
+                    extra_environ={'CKAN_LANG': lang_code,
+                                   'CKAN_CURRENT_URL': offset})
                 terms = ('A Novel By Tolstoy', tag_name, 'plain text', 'json')
                 for term in terms:
                     if term in translations:
                         assert translations[term] in response
-                    elif term in (
-                            ckan.lib.create_test_data.english_translations):
-                        assert ckan.lib.create_test_data.english_translations[term] in response
+                    elif term in _create_test_data.english_translations:
+                        assert (_create_test_data.english_translations[term]
+                                in response)
                     else:
                         assert term in response
                 assert 'this should not be rendered' not in response
@@ -123,55 +122,53 @@ def test_user_read_translation(self):
 
         '''
         for user_name in ('annafan',):
-            offset = routes.url_for(controller='user', action='read',
-                    id=user_name)
+            offset = routes.url_for(
+                controller='user', action='read', id=user_name)
             for (lang_code, translations) in (
-                    ('de', ckan.lib.create_test_data.german_translations),
-                    ('fr', ckan.lib.create_test_data.french_translations),
-                    ('en', ckan.lib.create_test_data.english_translations),
+                    ('de', _create_test_data.german_translations),
+                    ('fr', _create_test_data.french_translations),
+                    ('en', _create_test_data.english_translations),
                     ('pl', {})):
-                response = self.app.get(offset, status=200,
-                        extra_environ={'CKAN_LANG': lang_code,
-                            'CKAN_CURRENT_URL': offset})
+                response = self.app.get(
+                    offset,
+                    status=200,
+                    extra_environ={'CKAN_LANG': lang_code,
+                                   'CKAN_CURRENT_URL': offset})
                 terms = ('A Novel By Tolstoy', 'plain text', 'json')
                 for term in terms:
                     if term in translations:
                         assert translations[term] in response
-                    elif term in (
-                            ckan.lib.create_test_data.english_translations):
-                        assert ckan.lib.create_test_data.english_translations[term] in response
+                    elif term in _create_test_data.english_translations:
+                        assert (_create_test_data.english_translations[term]
+                                in response)
                     else:
                         assert term in response
                 assert 'this should not be rendered' not in response
 
     def test_group_read_translation(self):
         for (lang_code, translations) in (
-                ('de', ckan.lib.create_test_data.german_translations),
-                ('fr', ckan.lib.create_test_data.french_translations),
-                ('en', ckan.lib.create_test_data.english_translations),
+                ('de', _create_test_data.german_translations),
+                ('fr', _create_test_data.french_translations),
+                ('en', _create_test_data.english_translations),
                 ('pl', {})):
             offset = '/%s/group/roger' % lang_code
             response = self.app.get(offset, status=200)
             terms = ('A Novel By Tolstoy',
-                'Index of the novel',
-                'russian',
-                'tolstoy',
-                #"Dave's books",
-                "Roger's books",
-                #'Other (Open)',
-                #'romantic novel',
-                #'book',
-                '123',
-                '456',
-                '789',
-                'plain text',
-                'Roger likes these books.',
-            )
+                     'Index of the novel',
+                     'russian',
+                     'tolstoy',
+                     "Roger's books",
+                     '123',
+                     '456',
+                     '789',
+                     'plain text',
+                     'Roger likes these books.',)
             for term in terms:
                 if term in translations:
                     assert translations[term] in response
-                elif term in ckan.lib.create_test_data.english_translations:
-                    assert ckan.lib.create_test_data.english_translations[term] in response
+                elif term in _create_test_data.english_translations:
+                    assert (_create_test_data.english_translations[term]
+                            in response)
                 else:
                     assert term in response
             for tag_name in ('123', '456', '789', 'russian', 'tolstoy'):
@@ -180,31 +177,34 @@ def test_group_read_translation(self):
 
     def test_dataset_index_translation(self):
         for (lang_code, translations) in (
-                ('de', ckan.lib.create_test_data.german_translations),
-                ('fr', ckan.lib.create_test_data.french_translations),
-                ('en', ckan.lib.create_test_data.english_translations),
+                ('de', _create_test_data.german_translations),
+                ('fr', _create_test_data.french_translations),
+                ('en', _create_test_data.english_translations),
                 ('pl', {})):
             offset = '/%s/dataset' % lang_code
             response = self.app.get(offset, status=200)
             for term in ('Index of the novel', 'russian', 'tolstoy',
-                    "Dave's books", "Roger's books", 'plain text'):
+                         "Dave's books", "Roger's books", 'plain text'):
                 if term in translations:
                     assert translations[term] in response
-                elif term in ckan.lib.create_test_data.english_translations:
-                    assert ckan.lib.create_test_data.english_translations[term] in response
+                elif term in _create_test_data.english_translations:
+                    assert (_create_test_data.english_translations[term]
+                            in response)
                 else:
                     assert term in response
             for tag_name in ('123', '456', '789', 'russian', 'tolstoy'):
-                assert '/%s/dataset?tags=%s' % (lang_code, tag_name) in response
+                assert ('/%s/dataset?tags=%s' % (lang_code, tag_name)
+                        in response)
             for group_name in ('david', 'roger'):
-                assert '/%s/dataset?groups=%s' % (lang_code, group_name) in response
+                assert ('/%s/dataset?groups=%s' % (lang_code, group_name)
+                        in response)
             assert 'this should not be rendered' not in response
 
     def test_group_index_translation(self):
         for (lang_code, translations) in (
-                ('de', ckan.lib.create_test_data.german_translations),
-                ('fr', ckan.lib.create_test_data.french_translations),
-                ('en', ckan.lib.create_test_data.english_translations),
+                ('de', _create_test_data.german_translations),
+                ('fr', _create_test_data.french_translations),
+                ('en', _create_test_data.english_translations),
                 ('pl', {})):
             offset = '/%s/group' % lang_code
             response = self.app.get(offset, status=200)
@@ -217,8 +217,9 @@ def test_group_index_translation(self):
             for term in terms:
                 if term in translations:
                     assert translations[term] in response
-                elif term in ckan.lib.create_test_data.english_translations:
-                    assert ckan.lib.create_test_data.english_translations[term] in response
+                elif term in _create_test_data.english_translations:
+                    assert (_create_test_data.english_translations[term]
+                            in response)
                 else:
                     assert term in response
             for group_name in ('david', 'roger'):
@@ -227,9 +228,9 @@ def test_group_index_translation(self):
 
     def test_tag_index_translation(self):
         for (lang_code, translations) in (
-                ('de', ckan.lib.create_test_data.german_translations),
-                ('fr', ckan.lib.create_test_data.french_translations),
-                ('en', ckan.lib.create_test_data.english_translations),
+                ('de', _create_test_data.german_translations),
+                ('fr', _create_test_data.french_translations),
+                ('en', _create_test_data.english_translations),
                 ('pl', {})):
             offset = '/%s/tag' % lang_code
             response = self.app.get(offset, status=200)
@@ -243,13 +244,15 @@ def test_tag_index_translation(self):
             for term in terms:
                 if term in translations:
                     assert translations[term] in response
-                elif term in ckan.lib.create_test_data.english_translations:
-                    assert ckan.lib.create_test_data.english_translations[term] in response
+                elif term in _create_test_data.english_translations:
+                    assert (_create_test_data.english_translations[term]
+                            in response)
                 else:
                     assert term in response
                 assert '/%s/tag/%s' % (lang_code, term) in response
             assert 'this should not be rendered' not in response
 
+
 class TestDatasetSearchIndex():
 
     @classmethod
@@ -260,36 +263,28 @@ def setup_class(cls):
         data_dicts = [
             {'term': 'moo',
              'term_translation': 'french_moo',
-             'lang_code': 'fr',
-             }, # 
+             'lang_code': 'fr'},
             {'term': 'moo',
              'term_translation': 'this should not be rendered',
-             'lang_code': 'fsdas',
-             },
+             'lang_code': 'fsdas'},
             {'term': 'an interesting note',
              'term_translation': 'french note',
-             'lang_code': 'fr',
-             },
+             'lang_code': 'fr'},
             {'term': 'moon',
              'term_translation': 'french moon',
-             'lang_code': 'fr',
-             },
+             'lang_code': 'fr'},
             {'term': 'boon',
              'term_translation': 'french boon',
-             'lang_code': 'fr',
-             },
+             'lang_code': 'fr'},
             {'term': 'boon',
              'term_translation': 'italian boon',
-             'lang_code': 'it',
-             },
+             'lang_code': 'it'},
             {'term': 'david',
              'term_translation': 'french david',
-             'lang_code': 'fr',
-             },
+             'lang_code': 'fr'},
             {'term': 'david',
              'term_translation': 'italian david',
-             'lang_code': 'it',
-             },
+             'lang_code': 'it'}
         ]
 
         context = {
@@ -299,39 +294,41 @@ def setup_class(cls):
             'ignore_auth': True,
         }
         for data_dict in data_dicts:
-            ckan.logic.action.update.term_translation_update(context,
-                    data_dict)
+            ckan.logic.action.update.term_translation_update(
+                context, data_dict)
 
     @classmethod
     def teardown(cls):
         ckan.plugins.unload('multilingual_dataset')
         ckan.plugins.unload('multilingual_group')
 
-
     def test_translate_terms(self):
 
         sample_index_data = {
-         'download_url': u'moo',
-         'notes': u'an interesting note',
-         'tags': [u'moon', 'boon'],
-         'title': u'david',
-        } 
+            'download_url': u'moo',
+            'notes': u'an interesting note',
+            'tags': [u'moon', 'boon'],
+            'title': u'david',
+        }
 
-        result = mulilingual_plugin.MultilingualDataset().before_index(sample_index_data)
+        result = mulilingual_plugin.MultilingualDataset().before_index(
+            sample_index_data)
 
-        assert result == {'text_pl': '',
-                          'text_de': '',
-                          'text_ro': '',
-                          'title': u'david',
-                          'notes': u'an interesting note',
-                          'tags': [u'moon', 'boon'],
-                          'title_en': u'david',
-                          'download_url': u'moo',
-                          'text_it': u'italian boon',
-                          'text_es': '',
-                          'text_en': u'an interesting note moon boon moo',
-                          'text_nl': '',
-                          'title_it': u'italian david',
-                          'text_pt': '',
-                          'title_fr': u'french david',
-                          'text_fr': u'french note french boon french_moo french moon'}, result
+        assert result == {
+            'text_pl': '',
+            'text_de': '',
+            'text_ro': '',
+            'title': u'david',
+            'notes': u'an interesting note',
+            'tags': [u'moon', 'boon'],
+            'title_en': u'david',
+            'download_url': u'moo',
+            'text_it': u'italian boon',
+            'text_es': '',
+            'text_en': u'an interesting note moon boon moo',
+            'text_nl': '',
+            'title_it': u'italian david',
+            'text_pt': '',
+            'title_fr': u'french david',
+            'text_fr': u'french note french boon french_moo french moon'
+        }, result

From 9359e8d48fba6117caecbd990dafdb9321c68169 Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Wed, 23 Oct 2013 15:09:20 +0100
Subject: [PATCH 66/87] [#1200] fix more issues with pull request

---
 ckan/templates/package/resource_data.html |  2 +-
 ckanext/datapusher/logic/action.py        |  2 ++
 ckanext/datastore/logic/action.py         | 12 +++++-------
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/ckan/templates/package/resource_data.html b/ckan/templates/package/resource_data.html
index 4f19a6d11e6..05bd66fdfee 100644
--- a/ckan/templates/package/resource_data.html
+++ b/ckan/templates/package/resource_data.html
@@ -18,7 +18,7 @@
     <div class="alert alert-error">
       <strong>{{ _('Upload error:') }}</strong> {{ status.error.message }}
     </div>
-  {% elif status.task_info.error %}
+  {% elif status.task_info and status.task_info.error %}
     {% set show_table = false %}
     <div class="alert alert-error">
       <strong>{{ _('Error:') }}</strong> {{ status.task_info.error }}
diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py
index fb91a374243..1872a1ef265 100644
--- a/ckanext/datapusher/logic/action.py
+++ b/ckanext/datapusher/logic/action.py
@@ -186,11 +186,13 @@ def datapusher_status(context, data_dict):
     job_id = value.get('job_id')
     url = None
     job_detail = None
+
     if job_id:
         url = urlparse.urljoin(datapusher_url, 'job' + '/' + job_id)
         try:
             r = requests.get(url, headers={'Content-Type': 'application/json',
                                            'Authorization': job_key})
+            r.raise_for_status()
             job_detail = r.json()
         except (requests.exceptions.ConnectionError,
                 requests.exceptions.HTTPError), e:
diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
index 9e65f204712..29d9395e418 100644
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -96,15 +96,13 @@ def datastore_create(context, data_dict):
 
         # create resource from file
         if has_url:
-            try:
-                p.toolkit.get_action('datapusher_submit')(context, {
-                    'resource_id': res['id'],
-                    'set_url_type': True
-                })
-            except KeyError:
+            if not 'datapusher' in pylons.config.get('plugins'):
                 raise p.toolkit.ValidationError({'resource': [
                     'The datapusher has to be enabled.']})
-
+            p.toolkit.get_action('datapusher_submit')(context, {
+                'resource_id': res['id'],
+                'set_url_type': True
+            })
             # since we'll overwrite the datastore resource anyway, we
             # don't need to create it here
             return

From e09d4c3072a9a74d1ad30afd37da8b057c5eb619 Mon Sep 17 00:00:00 2001
From: John Martin <me@johnmart.in>
Date: Wed, 23 Oct 2013 16:07:40 +0100
Subject: [PATCH 67/87] [#1200] Fix for undefined  on new resource form on
 non-draft datasets

---
 ckan/templates/package/new_resource.html           |  7 +------
 ckan/templates/package/new_resource_not_draft.html | 12 ++++++++++++
 ckan/templates/package/resource_edit_base.html     |  6 ++++--
 ckan/templates/package/snippets/resource_help.html |  6 ++++++
 4 files changed, 23 insertions(+), 8 deletions(-)
 create mode 100644 ckan/templates/package/snippets/resource_help.html

diff --git a/ckan/templates/package/new_resource.html b/ckan/templates/package/new_resource.html
index 49d02387768..edd34bb5213 100644
--- a/ckan/templates/package/new_resource.html
+++ b/ckan/templates/package/new_resource.html
@@ -13,12 +13,7 @@
 {% block form %}{% snippet 'package/snippets/resource_form.html', data=data, errors=errors, error_summary=error_summary, include_metadata=false, pkg_name=pkg_name, stage=stage, allow_upload=g.ofs_impl and logged_in %}{% endblock %}
 
 {% block secondary_content %}
-  <section class="module module-narrow module-shallow">
-    <h2 class="module-heading"><i class="icon-info-sign"></i> {{ _('What\'s a resource?') }}</h2>
-    <div class="module-content">
-      <p>{{ _('A resource can be any file or link to a file containing useful data.') }}</p>
-    </div>
-  </section>
+  {% snippet 'package/snippets/resource_help.html' %}
 {% endblock %}
 
 {% block scripts %}
diff --git a/ckan/templates/package/new_resource_not_draft.html b/ckan/templates/package/new_resource_not_draft.html
index a24f37ce459..7e1638c51e7 100644
--- a/ckan/templates/package/new_resource_not_draft.html
+++ b/ckan/templates/package/new_resource_not_draft.html
@@ -3,6 +3,18 @@
 {% block subtitle %}{{ _('Add resource') }} - {{ h.dataset_display_name(pkg) }}{% endblock %}
 {% block form_title %}{{ _('Add resource') }}{% endblock %}
 
+{% block breadcrumb_content %}
+  <li class="active"><a href="#">{{ _('Add New Resource') }}</a></li>
+{% endblock %}
+
 {% block form %}
   {% snippet 'package/snippets/resource_form.html', data=data, errors=errors, error_summary=error_summary, include_metadata=false, pkg_name=pkg_name, stage=stage, allow_upload=g.ofs_impl and logged_in %}
 {% endblock %}
+
+{% block content_primary_nav %}
+  <li class="active"><a href="#"><i class="icon-edit"></i> {{ _('New resource') }}</a></li>
+{% endblock %}
+
+{% block secondary_content %}
+  {% snippet 'package/snippets/resource_help.html' %}
+{% endblock %}
diff --git a/ckan/templates/package/resource_edit_base.html b/ckan/templates/package/resource_edit_base.html
index 22faa8be7e1..0682cbbf014 100644
--- a/ckan/templates/package/resource_edit_base.html
+++ b/ckan/templates/package/resource_edit_base.html
@@ -8,12 +8,14 @@
 {% block breadcrumb_content %}
   {{ super() }}
   <li>{% link_for h.resource_display_name(res)|truncate(30), controller='package', action='resource_read', id=pkg.name, resource_id=res.id %}</li>
-  <li class="active"><a href="">Edit</a></li>
+  <li class="active"><a href="">{{ _('Edit') }}</a></li>
 {% endblock %}
 
 {% block content_action %}
   {% link_for _('All resources'), controller='package', action='resources', id=pkg.name, class_='btn', icon='arrow-left' %}
-  {% link_for _('View resource'), controller='package', action='resource_read', id=pkg.name, resource_id=res.id, class_='btn', icon='eye-open' %}
+  {% if res %}
+    {% link_for _('View resource'), controller='package', action='resource_read', id=pkg.name, resource_id=res.id, class_='btn', icon='eye-open' %}
+  {% endif %}
 {% endblock %}
 
 {% block content_primary_nav %}
diff --git a/ckan/templates/package/snippets/resource_help.html b/ckan/templates/package/snippets/resource_help.html
new file mode 100644
index 00000000000..d1724956909
--- /dev/null
+++ b/ckan/templates/package/snippets/resource_help.html
@@ -0,0 +1,6 @@
+<section class="module module-narrow module-shallow">
+  <h2 class="module-heading"><i class="icon-info-sign"></i> {{ _('What\'s a resource?') }}</h2>
+  <div class="module-content">
+    <p>{{ _('A resource can be any file or link to a file containing useful data.') }}</p>
+  </div>
+</section>

From 905e1fe5004fde3e764d9888546591f68cd0f079 Mon Sep 17 00:00:00 2001
From: joetsoi <joe.yeung.tsoi@gmail.com>
Date: Wed, 23 Oct 2013 17:09:26 +0100
Subject: [PATCH 68/87] [#1224] fix error messages in tests

---
 ckan/tests/functional/api/test_follow.py | 24 ++++++++++++++++--------
 ckan/tests/functional/test_related.py    |  3 ++-
 ckan/tests/logic/test_action.py          |  8 ++++++--
 ckan/tests/logic/test_tag.py             |  3 ++-
 4 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/ckan/tests/functional/api/test_follow.py b/ckan/tests/functional/api/test_follow.py
index 79e6f5dab17..d23ead94087 100644
--- a/ckan/tests/functional/api/test_follow.py
+++ b/ckan/tests/functional/api/test_follow.py
@@ -424,36 +424,42 @@ def test_01_user_follow_user_bad_apikey(self):
             error = ckan.tests.call_action_api(self.app, 'follow_user',
                     id=self.russianfan['id'], apikey=apikey,
                     status=403)
-            assert error['message'] == 'Access denied'
+            assert error['message'] == ('Access denied: '
+                'You must be logged in to follow users')
 
     def test_01_user_follow_dataset_bad_apikey(self):
         for apikey in ('bad api key', '', '     ', 'None', '3', '35.7', 'xxx'):
             error = ckan.tests.call_action_api(self.app, 'follow_dataset',
                     id=self.warandpeace['id'], apikey=apikey,
                     status=403)
-            assert error['message'] == 'Access denied'
+            assert error['message'] == ('Access denied: '
+                'You must be logged in to follow a dataset.')
 
     def test_01_user_follow_group_bad_apikey(self):
         for apikey in ('bad api key', '', '     ', 'None', '3', '35.7', 'xxx'):
             error = ckan.tests.call_action_api(self.app, 'follow_group',
                     id=self.rogers_group['id'], apikey=apikey,
                     status=403)
-            assert error['message'] == 'Access denied'
+            assert error['message'] == ('Access denied: '
+                'You must be logged in to follow a group.')
 
     def test_01_user_follow_user_missing_apikey(self):
         error = ckan.tests.call_action_api(self.app, 'follow_user',
                 id=self.russianfan['id'], status=403)
-        assert error['message'] == 'Access denied'
+        assert error['message'] == ('Access denied: '
+            'You must be logged in to follow users')
 
     def test_01_user_follow_dataset_missing_apikey(self):
         error = ckan.tests.call_action_api(self.app, 'follow_dataset',
                 id=self.warandpeace['id'], status=403)
-        assert error['message'] == 'Access denied'
+        assert error['message'] == ('Access denied: '
+            'You must be logged in to follow a dataset.')
 
     def test_01_user_follow_group_missing_apikey(self):
         error = ckan.tests.call_action_api(self.app, 'follow_group',
                 id=self.rogers_group['id'], status=403)
-        assert error['message'] == 'Access denied'
+        assert error['message'] == ('Access denied: '
+            'You must be logged in to follow a group.')
 
     def test_01_follow_bad_object_id(self):
         for action in ('follow_user', 'follow_dataset', 'follow_group'):
@@ -878,14 +884,16 @@ def test_01_unfollow_bad_apikey(self):
                     'xxx'):
                 error = ckan.tests.call_action_api(self.app, action,
                         apikey=apikey, status=403, id=self.joeadmin['id'])
-                assert error['message'] == 'Access denied'
+                assert error['message'] == ('Access denied: '
+                    'You must be logged in to unfollow something.')
 
     def test_01_unfollow_missing_apikey(self):
         '''Test error response when calling unfollow_* without api key.'''
         for action in ('unfollow_user', 'unfollow_dataset', 'unfollow_group'):
             error = ckan.tests.call_action_api(self.app, action, status=403,
                     id=self.joeadmin['id'])
-            assert error['message'] == 'Access denied'
+            assert error['message'] == ('Access denied: '
+                'You must be logged in to unfollow something.')
 
     def test_01_unfollow_bad_object_id(self):
         '''Test error response when calling unfollow_* with bad object id.'''
diff --git a/ckan/tests/functional/test_related.py b/ckan/tests/functional/test_related.py
index 003f0bb424b..5c82e8a8dba 100644
--- a/ckan/tests/functional/test_related.py
+++ b/ckan/tests/functional/test_related.py
@@ -498,4 +498,5 @@ def test_api_delete_fail(self):
                             extra_environ=extra)
         r = json.loads(res.body)
         assert r['success'] == False, r
-        assert r[u'error'][u'message'] == u'Access denied' , r
+        error_msg = u'Access denied: Only the owner can delete a related item'
+        assert r[u'error'][u'message'] == error_msg, r
diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py
index 4e496f7899c..1f89f778345 100644
--- a/ckan/tests/logic/test_action.py
+++ b/ckan/tests/logic/test_action.py
@@ -523,9 +523,11 @@ def test_12_user_update(self):
 
         res_obj = json.loads(res.body)
         assert res_obj['help'].startswith("Update a user account.")
+        error_msg = u'Access denied: User annafan not authorized to edit user {0}'.format(
+            sysadmin_user_dict['id'])
         assert res_obj['error'] == {
                 '__type': 'Authorization Error',
-                'message': 'Access denied'
+                'message': error_msg,
             }
         assert res_obj['success'] is False
 
@@ -787,7 +789,9 @@ def test_22_task_status_normal_user_not_authorized(self):
         res_obj = json.loads(res.body)
         assert res_obj['help'].startswith("Update a task status.")
         assert res_obj['success'] is False
-        assert res_obj['error'] == {'message': 'Access denied', '__type': 'Authorization Error'}
+        error_msg = (u'Access denied: User annafan not authorized to update '
+            'task_status table')
+        assert res_obj['error'] == {'message': error_msg, '__type': 'Authorization Error'}
 
     def test_23_task_status_validation(self):
         task_status = {}
diff --git a/ckan/tests/logic/test_tag.py b/ckan/tests/logic/test_tag.py
index 82d0cdd1e1e..7426fde827d 100644
--- a/ckan/tests/logic/test_tag.py
+++ b/ckan/tests/logic/test_tag.py
@@ -151,7 +151,8 @@ def test_08_user_create_not_authorized(self):
         res_obj = json.loads(res.body)
         assert res_obj['help'].startswith("Create a new user.")
         assert res_obj['success'] is False
-        assert res_obj['error'] == {'message': 'Access denied', '__type': 'Authorization Error'}
+        error_msg = 'Access denied: User  not authorized to create users'
+        assert res_obj['error'] == {'message': error_msg, '__type': 'Authorization Error'}
 
     def test_09_user_create(self):
         user_dict = {'name':'test_create_from_action_api',

From 05c27cefca4983d387ca1e66d0c381b9b9cbabfe Mon Sep 17 00:00:00 2001
From: Ian Ward <ian@excess.org>
Date: Wed, 23 Oct 2013 12:43:47 -0400
Subject: [PATCH 69/87] [#1281] call setup_template_variables in group/org
 read, about and bulk_process

---
 ckan/controllers/group.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py
index 66bd533fd43..f7ee98bef54 100644
--- a/ckan/controllers/group.py
+++ b/ckan/controllers/group.py
@@ -344,6 +344,9 @@ def pager_url(q=None, page=None):
             c.facets = {}
             c.page = h.Page(collection=[])
 
+        self._setup_template_variables(context, {'id':id},
+            group_type=group_type)
+
     def bulk_process(self, id):
         ''' Allow bulk processing of datasets for an organization.  Make
         private/public or delete. For organization admins.'''
@@ -828,7 +831,9 @@ def admins(self, id):
 
     def about(self, id):
         c.group_dict = self._get_group_dict(id)
-        return render(self._about_template(c.group_dict['type']))
+        group_type = c.group_dict['type']
+        self._setup_template_variables({}, {'id': id}, group_type=group_type)
+        return render(self._about_template(group_type))
 
     def _get_group_dict(self, id):
         ''' returns the result of group_show action or aborts if there is a

From 3be621a30f55aabb414abeadcf31c7b3141ca067 Mon Sep 17 00:00:00 2001
From: John Glover <j@johnglover.net>
Date: Thu, 24 Oct 2013 10:37:55 +0200
Subject: [PATCH 70/87] [#1274] Add tests for organization translation

---
 .../tests/test_multilingual_plugin.py         | 58 +++++++++++++++++++
 1 file changed, 58 insertions(+)

diff --git a/ckanext/multilingual/tests/test_multilingual_plugin.py b/ckanext/multilingual/tests/test_multilingual_plugin.py
index d7c4ec8a7f0..88041b54d59 100644
--- a/ckanext/multilingual/tests/test_multilingual_plugin.py
+++ b/ckanext/multilingual/tests/test_multilingual_plugin.py
@@ -3,6 +3,7 @@
 import ckan.lib.helpers
 import ckan.lib.create_test_data
 import ckan.logic.action.update
+import ckan.model as model
 import ckan.tests
 import ckan.tests.html_check
 import routes
@@ -22,6 +23,21 @@ def setup(cls):
         ckan.plugins.load('multilingual_tag')
         ckan.tests.setup_test_search_index()
         _create_test_data.CreateTestData.create_translations_test_data()
+
+        cls.sysadmin_user = model.User.get('testsysadmin')
+        cls.org = {'name': 'test_org',
+                   'title': 'russian',
+                   'description': 'Roger likes these books.'}
+        ckan.tests.call_action_api(cls.app, 'organization_create',
+                                   apikey=cls.sysadmin_user.apikey,
+                                   **cls.org)
+        dataset = {'name': 'test_org_dataset',
+                   'title': 'A Novel By Tolstoy',
+                   'owner_org': cls.org['name']}
+        ckan.tests.call_action_api(cls.app, 'package_create',
+                                   apikey=cls.sysadmin_user.apikey,
+                                   **dataset)
+
         # Add translation terms that match a couple of group names and package
         # names. Group names and package names should _not_ get translated even
         # if there are terms matching them, because they are used to form URLs.
@@ -175,6 +191,28 @@ def test_group_read_translation(self):
                 assert '%s?tags=%s' % (offset, tag_name) in response
             assert 'this should not be rendered' not in response
 
+    def test_org_read_translation(self):
+        for (lang_code, translations) in (
+                ('de', _create_test_data.german_translations),
+                ('fr', _create_test_data.french_translations),
+                ('en', _create_test_data.english_translations),
+                ('pl', {})):
+            offset = '/{0}/organization/{1}'.format(
+                lang_code, self.org['name'])
+            response = self.app.get(offset, status=200)
+            terms = ('A Novel By Tolstoy',
+                     'russian',
+                     'Roger likes these books.')
+            for term in terms:
+                if term in translations:
+                    assert translations[term] in response
+                elif term in _create_test_data.english_translations:
+                    assert (_create_test_data.english_translations[term]
+                            in response)
+                else:
+                    assert term in response
+            assert 'this should not be rendered' not in response
+
     def test_dataset_index_translation(self):
         for (lang_code, translations) in (
                 ('de', _create_test_data.german_translations),
@@ -226,6 +264,26 @@ def test_group_index_translation(self):
                 assert '/%s/group/%s' % (lang_code, group_name) in response
             assert 'this should not be rendered' not in response
 
+    def test_org_index_translation(self):
+        for (lang_code, translations) in (
+                ('de', _create_test_data.german_translations),
+                ('fr', _create_test_data.french_translations),
+                ('en', _create_test_data.english_translations),
+                ('pl', {})):
+            offset = '/{0}/organization'.format(lang_code)
+            response = self.app.get(offset, status=200)
+            for term in ('russian', 'Roger likes these books.'):
+                if term in translations:
+                    assert translations[term] in response
+                elif term in _create_test_data.english_translations:
+                    assert (_create_test_data.english_translations[term]
+                            in response)
+                else:
+                    assert term in response, response
+            assert ('/{0}/organization/{1}'.format(lang_code, self.org['name'])
+                    in response)
+            assert 'this should not be rendered' not in response
+
     def test_tag_index_translation(self):
         for (lang_code, translations) in (
                 ('de', _create_test_data.german_translations),

From ca51350e25f0350f263bc426c395e5a7686ed759 Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Thu, 24 Oct 2013 10:03:33 +0100
Subject: [PATCH 71/87] [#1200] fix error in getting plugings from config

---
 ckanext/datastore/logic/action.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
index 29d9395e418..aadf896dbe4 100644
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -96,7 +96,7 @@ def datastore_create(context, data_dict):
 
         # create resource from file
         if has_url:
-            if not 'datapusher' in pylons.config.get('plugins'):
+            if not 'datapusher' in pylons.config.get('ckan.plugins', ''):
                 raise p.toolkit.ValidationError({'resource': [
                     'The datapusher has to be enabled.']})
             p.toolkit.get_action('datapusher_submit')(context, {

From 1fa0e88579ec3b7d5fef9d07cb20c7880ea1dee6 Mon Sep 17 00:00:00 2001
From: John Martin <me@johnmart.in>
Date: Thu, 24 Oct 2013 10:17:44 +0100
Subject: [PATCH 72/87] [#1200] Fix for failing mocha test it's because of
 scoping...

---
 ckan/public/base/javascript/main.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ckan/public/base/javascript/main.js b/ckan/public/base/javascript/main.js
index 57331d782e6..42c76dfd15f 100644
--- a/ckan/public/base/javascript/main.js
+++ b/ckan/public/base/javascript/main.js
@@ -36,7 +36,7 @@ this.ckan = this.ckan || {};
       ckan.i18n.load(data);
       ckan.module.initialize();
     });
-    $('[data-target="popover"').popover();
+    jQuery('[data-target="popover"]').popover();
   };
 
   /* Returns a full url for the current site with the provided path appended.

From 58ccf52ee94c1650944f2481e4ab91a829f3a07a Mon Sep 17 00:00:00 2001
From: John Glover <j@johnglover.net>
Date: Thu, 24 Oct 2013 12:27:28 +0200
Subject: [PATCH 73/87] [#504] PEP8, and verify that items in original
 related_item dict are still present after the partial update.

---
 ckan/tests/logic/test_action.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py
index fac25a7ba86..c295d9271ae 100644
--- a/ckan/tests/logic/test_action.py
+++ b/ckan/tests/logic/test_action.py
@@ -2013,7 +2013,7 @@ def _add_basic_package(self, package_name=u'test_package', **kwargs):
                             extra_environ={'Authorization': 'tester'})
         return json.loads(res.body)['result']
 
-    def test_01_update_add_related_item(self):
+    def test_update_add_related_item(self):
         package = self._add_basic_package()
         related_item = {
             "description": "Testing a Description",
@@ -2025,17 +2025,22 @@ def test_01_update_add_related_item(self):
             "dataset_id": package['id'],
         }
         related_item_json = json.dumps(related_item)
-        res_create = self.app.post('/api/action/related_create', params=related_item_json,
-                            extra_environ={'Authorization': 'tester'})
-        assert res_create.json['success'] == True
+        res_create = self.app.post('/api/action/related_create',
+                                   params=related_item_json,
+                                   extra_environ={'Authorization': 'tester'})
+        assert res_create.json['success']
 
         related_update = res_create.json['result']
         related_update = {'id': related_update['id'], 'title': 'Updated'}
         related_update_json = json.dumps(related_update)
-        res_update = self.app.post('/api/action/related_update', params=related_update_json,
-                            extra_environ={'Authorization': 'tester'})
-        assert res_update.json['success'] == True
+        res_update = self.app.post('/api/action/related_update',
+                                   params=related_update_json,
+                                   extra_environ={'Authorization': 'tester'})
+        assert res_update.json['success']
         res_update_json = res_update.json['result']
         assert res_update_json['title'] == related_update['title']
 
-
+        related_item.pop('title')
+        related_item.pop('dataset_id')
+        for field in related_item:
+            assert related_item[field] == res_update_json[field]

From faf01bcdaad186db98abacd9de0769c2abf61448 Mon Sep 17 00:00:00 2001
From: joetsoi <joe.yeung.tsoi@gmail.com>
Date: Thu, 24 Oct 2013 11:36:11 +0100
Subject: [PATCH 74/87] [#1224] change tests to check for error type instead of
 message

---
 .../test_group_and_organization_purge.py      |  3 +--
 .../functional/api/model/test_vocabulary.py   | 22 ++++++++---------
 ckan/tests/functional/api/test_follow.py      | 24 +++++++------------
 ckan/tests/functional/test_related.py         |  3 +--
 ckan/tests/logic/test_action.py               | 11 ++-------
 ckan/tests/logic/test_tag.py                  |  3 +--
 .../tests/test_example_iauthfunctions.py      | 12 ++++------
 7 files changed, 28 insertions(+), 50 deletions(-)

diff --git a/ckan/tests/functional/api/model/test_group_and_organization_purge.py b/ckan/tests/functional/api/model/test_group_and_organization_purge.py
index 0770642dbe3..88a44e494f0 100644
--- a/ckan/tests/functional/api/model/test_group_and_organization_purge.py
+++ b/ckan/tests/functional/api/model/test_group_and_organization_purge.py
@@ -266,8 +266,7 @@ def _test_visitors_cannot_purge_groups_or_orgs(self, is_org):
         result = tests.call_action_api(self.app, action, id=group_or_org['id'],
                                        status=403,
                                        )
-        assert result == {'__type': 'Authorization Error',
-                          'message': 'Access denied'}
+        assert result['__type'] == 'Authorization Error'
 
     def test_visitors_cannot_purge_organizations(self):
         '''Visitors (who aren't logged in) should not be authorized to purge
diff --git a/ckan/tests/functional/api/model/test_vocabulary.py b/ckan/tests/functional/api/model/test_vocabulary.py
index 5330fb09822..83bd8a57676 100644
--- a/ckan/tests/functional/api/model/test_vocabulary.py
+++ b/ckan/tests/functional/api/model/test_vocabulary.py
@@ -399,7 +399,7 @@ def test_vocabulary_create_not_logged_in(self):
                 params=param_string,
                 status=403)
         assert response.json['success'] is False
-        assert response.json['error']['message'] == 'Access denied'
+        assert response.json['error']['__type'] == 'Authorization Error'
 
     def test_vocabulary_create_not_authorized(self):
         '''Test that users who are not authorized cannot create vocabs.'''
@@ -412,7 +412,7 @@ def test_vocabulary_create_not_authorized(self):
                     str(self.normal_user.apikey)},
                 status=403)
         assert response.json['success'] is False
-        assert response.json['error']['message'] == 'Access denied'
+        assert response.json['error']['__type'] == 'Authorization Error'
 
     def test_vocabulary_update_id_only(self):
         self._update_vocabulary({'id': self.genre_vocab['id']},
@@ -494,7 +494,7 @@ def test_vocabulary_update_not_logged_in(self):
                 params=param_string,
                 status=403)
         assert response.json['success'] is False
-        assert response.json['error']['message'] == 'Access denied'
+        assert response.json['error']['__type'] == 'Authorization Error'
 
     def test_vocabulary_update_with_tags(self):
         tags = [
@@ -630,7 +630,7 @@ def test_vocabulary_delete_not_logged_in(self):
                 params=param_string,
                 status=403)
         assert response.json['success'] is False
-        assert response.json['error']['message'] == 'Access denied'
+        assert response.json['error']['__type'] == 'Authorization Error'
 
     def test_vocabulary_delete_not_authorized(self):
         '''Test that users who are not authorized cannot delete vocabs.'''
@@ -642,7 +642,7 @@ def test_vocabulary_delete_not_authorized(self):
                     str(self.normal_user.apikey)},
                 status=403)
         assert response.json['success'] is False
-        assert response.json['error']['message'] == 'Access denied'
+        assert response.json['error']['__type'] == 'Authorization Error'
 
     def test_add_tag_to_vocab(self):
         '''Test that a tag can be added to and then retrieved from a vocab.'''
@@ -781,7 +781,7 @@ def test_add_tag_not_logged_in(self):
                 params=tag_string,
                 status=403)
         assert response.json['success'] is False
-        assert response.json['error']['message'] == 'Access denied'
+        assert response.json['error']['__type'] == 'Authorization Error'
 
     def test_add_tag_not_authorized(self):
         tag_dict = {
@@ -795,7 +795,7 @@ def test_add_tag_not_authorized(self):
                         str(self.normal_user.apikey)},
                 status=403)
         assert response.json['success'] is False
-        assert response.json['error']['message'] == 'Access denied'
+        assert response.json['error']['__type'] == 'Authorization Error'
 
     def test_add_vocab_tag_to_dataset(self):
         '''Test that a tag belonging to a vocab can be added to a dataset,
@@ -1116,8 +1116,8 @@ def test_delete_tag_not_logged_in(self):
                 params=helpers.json.dumps(params),
                 status=403)
         assert response.json['success'] is False
-        msg = response.json['error']['message']
-        assert msg == u"Access denied", msg
+        error = response.json['error']['__type']
+        assert error == u"Authorization Error", error
 
     def test_delete_tag_not_authorized(self):
         vocab = self.genre_vocab
@@ -1131,5 +1131,5 @@ def test_delete_tag_not_authorized(self):
                     str(self.normal_user.apikey)},
                 status=403)
         assert response.json['success'] is False
-        msg = response.json['error']['message']
-        assert msg == u"Access denied"
+        msg = response.json['error']['__type']
+        assert msg == u"Authorization Error"
diff --git a/ckan/tests/functional/api/test_follow.py b/ckan/tests/functional/api/test_follow.py
index d23ead94087..74d8cc75b88 100644
--- a/ckan/tests/functional/api/test_follow.py
+++ b/ckan/tests/functional/api/test_follow.py
@@ -424,42 +424,36 @@ def test_01_user_follow_user_bad_apikey(self):
             error = ckan.tests.call_action_api(self.app, 'follow_user',
                     id=self.russianfan['id'], apikey=apikey,
                     status=403)
-            assert error['message'] == ('Access denied: '
-                'You must be logged in to follow users')
+            assert error['__type'] == 'Authorization Error'
 
     def test_01_user_follow_dataset_bad_apikey(self):
         for apikey in ('bad api key', '', '     ', 'None', '3', '35.7', 'xxx'):
             error = ckan.tests.call_action_api(self.app, 'follow_dataset',
                     id=self.warandpeace['id'], apikey=apikey,
                     status=403)
-            assert error['message'] == ('Access denied: '
-                'You must be logged in to follow a dataset.')
+            assert error['__type'] == 'Authorization Error'
 
     def test_01_user_follow_group_bad_apikey(self):
         for apikey in ('bad api key', '', '     ', 'None', '3', '35.7', 'xxx'):
             error = ckan.tests.call_action_api(self.app, 'follow_group',
                     id=self.rogers_group['id'], apikey=apikey,
                     status=403)
-            assert error['message'] == ('Access denied: '
-                'You must be logged in to follow a group.')
+            assert error['__type'] == 'Authorization Error'
 
     def test_01_user_follow_user_missing_apikey(self):
         error = ckan.tests.call_action_api(self.app, 'follow_user',
                 id=self.russianfan['id'], status=403)
-        assert error['message'] == ('Access denied: '
-            'You must be logged in to follow users')
+        assert error['__type'] == 'Authorization Error'
 
     def test_01_user_follow_dataset_missing_apikey(self):
         error = ckan.tests.call_action_api(self.app, 'follow_dataset',
                 id=self.warandpeace['id'], status=403)
-        assert error['message'] == ('Access denied: '
-            'You must be logged in to follow a dataset.')
+        assert error['__type'] == 'Authorization Error'
 
     def test_01_user_follow_group_missing_apikey(self):
         error = ckan.tests.call_action_api(self.app, 'follow_group',
                 id=self.rogers_group['id'], status=403)
-        assert error['message'] == ('Access denied: '
-            'You must be logged in to follow a group.')
+        assert error['__type'] == 'Authorization Error'
 
     def test_01_follow_bad_object_id(self):
         for action in ('follow_user', 'follow_dataset', 'follow_group'):
@@ -884,16 +878,14 @@ def test_01_unfollow_bad_apikey(self):
                     'xxx'):
                 error = ckan.tests.call_action_api(self.app, action,
                         apikey=apikey, status=403, id=self.joeadmin['id'])
-                assert error['message'] == ('Access denied: '
-                    'You must be logged in to unfollow something.')
+                assert error['__type'] == 'Authorization Error'
 
     def test_01_unfollow_missing_apikey(self):
         '''Test error response when calling unfollow_* without api key.'''
         for action in ('unfollow_user', 'unfollow_dataset', 'unfollow_group'):
             error = ckan.tests.call_action_api(self.app, action, status=403,
                     id=self.joeadmin['id'])
-            assert error['message'] == ('Access denied: '
-                'You must be logged in to unfollow something.')
+            assert error['__type'] == 'Authorization Error'
 
     def test_01_unfollow_bad_object_id(self):
         '''Test error response when calling unfollow_* with bad object id.'''
diff --git a/ckan/tests/functional/test_related.py b/ckan/tests/functional/test_related.py
index 5c82e8a8dba..6fa2fed4266 100644
--- a/ckan/tests/functional/test_related.py
+++ b/ckan/tests/functional/test_related.py
@@ -498,5 +498,4 @@ def test_api_delete_fail(self):
                             extra_environ=extra)
         r = json.loads(res.body)
         assert r['success'] == False, r
-        error_msg = u'Access denied: Only the owner can delete a related item'
-        assert r[u'error'][u'message'] == error_msg, r
+        assert r[u'error'][u'__type'] == "Authorization Error", r
diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py
index bfc9c02f22d..d7c37b98aa7 100644
--- a/ckan/tests/logic/test_action.py
+++ b/ckan/tests/logic/test_action.py
@@ -528,12 +528,7 @@ def test_12_user_update(self):
 
         res_obj = json.loads(res.body)
         assert res_obj['help'].startswith("Update a user account.")
-        error_msg = u'Access denied: User annafan not authorized to edit user {0}'.format(
-            sysadmin_user_dict['id'])
-        assert res_obj['error'] == {
-                '__type': 'Authorization Error',
-                'message': error_msg,
-            }
+        assert res_obj['error']['__type'] == 'Authorization Error'
         assert res_obj['success'] is False
 
     def test_12_user_update_errors(self):
@@ -895,9 +890,7 @@ def test_22_task_status_normal_user_not_authorized(self):
         res_obj = json.loads(res.body)
         assert res_obj['help'].startswith("Update a task status.")
         assert res_obj['success'] is False
-        error_msg = (u'Access denied: User annafan not authorized to update '
-            'task_status table')
-        assert res_obj['error'] == {'message': error_msg, '__type': 'Authorization Error'}
+        assert res_obj['error']['__type'] == 'Authorization Error'
 
     def test_23_task_status_validation(self):
         task_status = {}
diff --git a/ckan/tests/logic/test_tag.py b/ckan/tests/logic/test_tag.py
index 7426fde827d..4844fa055ca 100644
--- a/ckan/tests/logic/test_tag.py
+++ b/ckan/tests/logic/test_tag.py
@@ -151,8 +151,7 @@ def test_08_user_create_not_authorized(self):
         res_obj = json.loads(res.body)
         assert res_obj['help'].startswith("Create a new user.")
         assert res_obj['success'] is False
-        error_msg = 'Access denied: User  not authorized to create users'
-        assert res_obj['error'] == {'message': error_msg, '__type': 'Authorization Error'}
+        assert res_obj['error']['__type'] == 'Authorization Error'
 
     def test_09_user_create(self):
         user_dict = {'name':'test_create_from_action_api',
diff --git a/ckanext/example_iauthfunctions/tests/test_example_iauthfunctions.py b/ckanext/example_iauthfunctions/tests/test_example_iauthfunctions.py
index 49b250c62b0..7031867df27 100644
--- a/ckanext/example_iauthfunctions/tests/test_example_iauthfunctions.py
+++ b/ckanext/example_iauthfunctions/tests/test_example_iauthfunctions.py
@@ -103,8 +103,7 @@ def test_group_create_with_visitor(self):
         result = tests.call_action_api(self.app, 'group_create',
                                        name='this_group_should_not_be_created',
                                        status=403)
-        assert result == {'__type': 'Authorization Error',
-                          'message': 'Access denied'}
+        assert result['__type'] == 'Authorization Error'
 
     def test_group_create_with_non_curator(self):
         '''A user who isn't a member of the curators group should not be able
@@ -116,8 +115,7 @@ def test_group_create_with_non_curator(self):
                                        name='this_group_should_not_be_created',
                                        apikey=noncurator['apikey'],
                                        status=403)
-        assert result == {'__type': 'Authorization Error',
-                          'message': 'Access denied'}
+        assert result['__type'] == 'Authorization Error'
 
     def test_group_create_with_curator(self):
         '''A member of the curators group should be able to create a group.
@@ -174,8 +172,7 @@ def test_group_create_with_visitor(self):
         response = tests.call_action_api(self.app, 'group_create',
                                          name='this_group_shouldnt_be_created',
                                          status=403)
-        assert response == {'__type': 'Authorization Error',
-                            'message': 'Access denied'}
+        assert response['__type'] == 'Authorization Error'
 
 
 class TestExampleIAuthFunctionsPluginV2(TestExampleIAuthFunctionsPlugin):
@@ -203,5 +200,4 @@ def test_group_create_with_curator(self):
                                        name='this_group_should_not_be_created',
                                        apikey=curator['apikey'],
                                        status=403)
-        assert result == {'__type': 'Authorization Error',
-                          'message': 'Access denied'}
+        assert result['__type'] == 'Authorization Error'

From f18f76bf13468e709fd7af4b763e824f69376292 Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Thu, 24 Oct 2013 19:14:36 +0100
Subject: [PATCH 75/87] [#1200] add loaded plugin logic

---
 ckan/plugins/core.py              | 11 ++++++++++-
 ckanext/datastore/logic/action.py |  2 +-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/ckan/plugins/core.py b/ckan/plugins/core.py
index 8f3752117ea..f39c8062917 100644
--- a/ckan/plugins/core.py
+++ b/ckan/plugins/core.py
@@ -18,7 +18,7 @@
     'PluginNotFoundException', 'Plugin', 'SingletonPlugin',
     'load', 'load_all', 'unload', 'unload_all',
     'get_plugin', 'plugins_update',
-    'use_plugin',
+    'use_plugin', 'plugin_loaded',
 ]
 
 log = logging.getLogger(__name__)
@@ -210,6 +210,15 @@ def unload(*plugins):
     plugins_update()
 
 
+def plugin_loaded(name):
+    '''
+    See if a particular plugin is loaded.
+    '''
+    if name in _PLUGINS:
+        return True
+    return False
+
+
 def find_system_plugins():
     '''
     Return all plugins in the ckan.system_plugins entry point group.
diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
index aadf896dbe4..804fec1a197 100644
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -96,7 +96,7 @@ def datastore_create(context, data_dict):
 
         # create resource from file
         if has_url:
-            if not 'datapusher' in pylons.config.get('ckan.plugins', ''):
+            if not p.plugin_loaded('datapusher'):
                 raise p.toolkit.ValidationError({'resource': [
                     'The datapusher has to be enabled.']})
             p.toolkit.get_action('datapusher_submit')(context, {

From c22b09089682ed547e53b9f7710ff93502d1fcbe Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 25 Oct 2013 13:20:08 +0100
Subject: [PATCH 76/87] [#1295] Don't return private datasets on package_list

---
 ckan/logic/action/get.py        |  7 +++++--
 ckan/tests/logic/test_action.py | 30 ++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py
index 70326e75317..ca46ac2ff57 100644
--- a/ckan/logic/action/get.py
+++ b/ckan/logic/action/get.py
@@ -91,8 +91,11 @@ def package_list(context, data_dict):
     col = (package_revision_table.c.id
         if api == 2 else package_revision_table.c.name)
     query = _select([col])
-    query = query.where(_and_(package_revision_table.c.state=='active',
-        package_revision_table.c.current==True))
+    query = query.where(_and_(
+                package_revision_table.c.state=='active',
+                package_revision_table.c.current==True,
+                package_revision_table.c.private==False,
+                ))
     query = query.order_by(col)
 
     limit = data_dict.get('limit')
diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py
index d7c37b98aa7..40b1a42d85d 100644
--- a/ckan/tests/logic/test_action.py
+++ b/ckan/tests/logic/test_action.py
@@ -84,6 +84,36 @@ def test_01_package_list(self):
         assert 'warandpeace' in res['result']
         assert 'annakarenina' in res['result']
 
+    def test_01_package_list_private(self):
+        tests.call_action_api(self.app, 'organization_create',
+                                        name='test_org_2',
+                                        apikey=self.sysadmin_user.apikey)
+
+        tests.call_action_api(self.app, 'package_create',
+                                        name='public_dataset',
+                                        owner_org='test_org_2',
+                                        apikey=self.sysadmin_user.apikey)
+
+        res = tests.call_action_api(self.app, 'package_list')
+
+        assert len(res) == 3
+        assert 'warandpeace' in res
+        assert 'annakarenina' in res
+        assert 'public_dataset' in res
+
+        tests.call_action_api(self.app, 'package_create',
+                                        name='private_dataset',
+                                        owner_org='test_org_2',
+                                        private=True,
+                                        apikey=self.sysadmin_user.apikey)
+
+        res = tests.call_action_api(self.app, 'package_list')
+        assert len(res) == 3
+        assert 'warandpeace' in res
+        assert 'annakarenina' in res
+        assert 'public_dataset' in res
+        assert not 'private_dataset' in res
+
     def test_01_current_package_list_with_resources(self):
         url = '/api/action/current_package_list_with_resources'
 

From 500733eeb791b3960e0a7b116499ad8a27d207cf Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Tue, 29 Oct 2013 11:21:29 +0000
Subject: [PATCH 77/87] Default API_URL to SITE_URL if missing

Otherwise file uploads and other calls where broken when using CKAN in
a non-root location.
---
 ckan/public/base/javascript/client.js | 2 +-
 ckan/public/base/javascript/main.js   | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/ckan/public/base/javascript/client.js b/ckan/public/base/javascript/client.js
index 45940c0086d..e1c60ccf24e 100644
--- a/ckan/public/base/javascript/client.js
+++ b/ckan/public/base/javascript/client.js
@@ -370,7 +370,7 @@
   });
 
   ckan.sandbox.setup(function (instance) {
-    instance.client = new Client({endpoint: ckan.API_ROOT});
+    instance.client = new Client({endpoint: ckan.SITE_ROOT});
   });
 
   ckan.Client = Client;
diff --git a/ckan/public/base/javascript/main.js b/ckan/public/base/javascript/main.js
index 42c76dfd15f..843dc47442d 100644
--- a/ckan/public/base/javascript/main.js
+++ b/ckan/public/base/javascript/main.js
@@ -29,7 +29,9 @@ this.ckan = this.ckan || {};
 
     ckan.SITE_ROOT   = getRootFromData('siteRoot');
     ckan.LOCALE_ROOT = getRootFromData('localeRoot');
-    ckan.API_ROOT    = getRootFromData('apiRoot');
+
+    // Not used, kept for backwards compatibility
+    ckan.API_ROOT    = body.data('apiRoot') || ckan.SITE_ROOT;
 
     // Load the localisations before instantiating the modules.
     ckan.sandbox().client.getLocaleData(locale).done(function (data) {

From 787cb7ae25ce47bfa28b8543c1a7e59a06186fe4 Mon Sep 17 00:00:00 2001
From: Nigel Babu <nigelbabu@gmail.com>
Date: Fri, 1 Nov 2013 13:35:59 +0530
Subject: [PATCH 78/87] Fix a few small mistakes in the tests

---
 ckan/tests/functional/api/test_user.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/ckan/tests/functional/api/test_user.py b/ckan/tests/functional/api/test_user.py
index cdbe039d012..e313e29710c 100644
--- a/ckan/tests/functional/api/test_user.py
+++ b/ckan/tests/functional/api/test_user.py
@@ -133,7 +133,7 @@ def teardown_class(cls):
 
         model.repo.rebuild_db()
 
-    def test_user_create_api_disabled_sysadmin(self):
+    def test_user_create_api_enabled_sysadmin(self):
         params = {
             'name': 'testinganewusersysadmin',
             'email': 'testinganewuser@ckan.org',
@@ -145,14 +145,13 @@ def test_user_create_api_disabled_sysadmin(self):
         res_dict = res.json
         assert res_dict['success'] is True
 
-    def test_user_create_api_disabled_anon(self):
+    def test_user_create_api_enabled_anon(self):
         params = {
             'name': 'testinganewuseranon',
             'email': 'testinganewuser@ckan.org',
             'password': 'random',
         }
-        res = self.app.post('/api/3/action/user_create', json.dumps(params),
-                expect_errors=True)
+        res = self.app.post('/api/3/action/user_create', json.dumps(params))
         res_dict = res.json
         assert res_dict['success'] is True
 

From 4f732627fbf515bc7cbe29e84731606409a637b7 Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Fri, 1 Nov 2013 14:47:41 +0000
Subject: [PATCH 79/87] [#1262] only allow file already exists errors to not
 raise exception

---
 ckan/lib/uploader.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py
index 867e28cfcb0..cc4e8751d57 100644
--- a/ckan/lib/uploader.py
+++ b/ckan/lib/uploader.py
@@ -57,7 +57,9 @@ def __init__(self, object_type, old_filename=None):
         try:
             os.makedirs(self.storage_path)
         except OSError, e:
-            pass
+            ## errno 17 is file already exists
+            if e.errno != 17:
+                raise
         self.object_type = object_type
         self.old_filename = old_filename
         if old_filename:

From 8c7329c6246c03a965f8284c855c3e538d205a9c Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Mon, 4 Nov 2013 07:45:40 +0000
Subject: [PATCH 80/87] [#1262] only allow file already exists errors to not
 raise exception for other case

---
 ckan/config/middleware.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py
index efde540054a..5ba02933d46 100644
--- a/ckan/config/middleware.py
+++ b/ckan/config/middleware.py
@@ -155,7 +155,9 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf):
             try:
                 os.makedirs(path)
             except OSError, e:
-                pass
+                ## errno 17 is file already exists
+                if e.errno != 17:
+                    raise
 
             storage_app = StaticURLParser(path,
                 cache_max_age=static_max_age)

From 8b15c7938295f7919d95857975d45282f3daffbe Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Mon, 4 Nov 2013 14:23:02 +0000
Subject: [PATCH 81/87] [#1262] fix pull request issues

---
 ckan/controllers/group.py       |  4 --
 ckan/public/base/css/main.css   | 78 +++++++++++++++++----------------
 ckan/templates/macros/form.html |  6 +--
 3 files changed, 44 insertions(+), 44 deletions(-)

diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py
index adbd3d7cacc..907bda73c41 100644
--- a/ckan/controllers/group.py
+++ b/ckan/controllers/group.py
@@ -7,7 +7,6 @@
 from urllib import urlencode
 
 from pylons.i18n import get_lang
-from pylons import config
 
 import ckan.lib.base as base
 import ckan.lib.helpers as h
@@ -462,7 +461,6 @@ def edit(self, id, data=None, errors=None, error_summary=None):
 
         group = context.get("group")
         c.group = group
-        context.pop('for_edit')
         c.group_dict = self._action('group_show')(context, data_dict)
 
         try:
@@ -521,8 +519,6 @@ def _force_reindex(self, grp):
 
     def _save_edit(self, id, context):
         try:
-            old_group = self._action('group_show')(context, {"id": id})
-            old_image_url = old_group.get('image_url')
             data_dict = clean_dict(dict_fns.unflatten(
                 tuplize_dict(parse_params(request.params))))
             context['message'] = data_dict.get('log_message', '')
diff --git a/ckan/public/base/css/main.css b/ckan/public/base/css/main.css
index 2ed06e1eb12..0fbaa417e14 100644
--- a/ckan/public/base/css/main.css
+++ b/ckan/public/base/css/main.css
@@ -49,16 +49,12 @@ sub {
 }
 img {
   /* Responsive images (ensure images don't scale beyond their parents) */
-
   max-width: 100%;
   /* Part 1: Set a maxium relative to the parent */
-
   width: auto\9;
   /* IE7-8 need help adjusting responsive images */
-
   height: auto;
   /* Part 2: Scale the height according to the width, otherwise you get stretching */
-
   vertical-align: middle;
   border: 0;
   -ms-interpolation-mode: bicubic;
@@ -153,7 +149,7 @@ textarea {
   img {
     max-width: 100% !important;
   }
-  @page  {
+  @page {
     margin: 0.5cm;
   }
   p,
@@ -693,7 +689,6 @@ ol.inline > li {
   display: inline-block;
   *display: inline;
   /* IE7 inline-block hack */
-
   *zoom: 1;
   padding-left: 5px;
   padding-right: 5px;
@@ -972,7 +967,6 @@ input[type="color"]:focus,
   outline: 0;
   outline: thin dotted \9;
   /* IE6-9 */
-
   -webkit-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6);
   -moz-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6);
   box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6);
@@ -982,10 +976,8 @@ input[type="checkbox"] {
   margin: 4px 0 0;
   *margin-top: 0;
   /* IE7 */
-
   margin-top: 1px \9;
   /* IE8-9 */
-
   line-height: normal;
 }
 input[type="file"],
@@ -1001,10 +993,8 @@ select,
 input[type="file"] {
   height: 30px;
   /* In IE7, the height of the select element cannot be changed by height, only font-size */
-
   *margin-top: 4px;
   /* For IE7, add top margin to align select with labels */
-
   line-height: 30px;
 }
 select {
@@ -1402,7 +1392,6 @@ select:focus:invalid:focus {
   display: inline-block;
   *display: inline;
   /* IE7 inline-block hack */
-
   *zoom: 1;
   vertical-align: middle;
   padding-left: 5px;
@@ -1553,7 +1542,6 @@ input.search-query {
   padding-left: 14px;
   padding-left: 4px \9;
   /* IE7-8 doesn't have border-radius, so don't indent the padding */
-
   margin-bottom: 0;
   -webkit-border-radius: 15px;
   -moz-border-radius: 15px;
@@ -1610,7 +1598,6 @@ input.search-query {
   display: inline-block;
   *display: inline;
   /* IE7 inline-block hack */
-
   *zoom: 1;
   margin-bottom: 0;
   vertical-align: middle;
@@ -2220,7 +2207,6 @@ button.close {
   display: inline-block;
   *display: inline;
   /* IE7 inline-block hack */
-
   *zoom: 1;
   padding: 4px 12px;
   margin-bottom: 0;
@@ -2243,7 +2229,6 @@ button.close {
   border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
   *background-color: #eaeaea;
   /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
   filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
   border: 1px solid #cccccc;
   *border: 0;
@@ -2379,7 +2364,6 @@ input[type="button"].btn-block {
   border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
   *background-color: #085871;
   /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
   filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
 }
 .btn-primary:hover,
@@ -2411,7 +2395,6 @@ input[type="button"].btn-block {
   border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
   *background-color: #f89406;
   /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
   filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
 }
 .btn-warning:hover,
@@ -2443,7 +2426,6 @@ input[type="button"].btn-block {
   border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
   *background-color: #bd362f;
   /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
   filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
 }
 .btn-danger:hover,
@@ -2475,7 +2457,6 @@ input[type="button"].btn-block {
   border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
   *background-color: #51a351;
   /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
   filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
 }
 .btn-success:hover,
@@ -2507,7 +2488,6 @@ input[type="button"].btn-block {
   border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
   *background-color: #2f96b4;
   /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
   filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
 }
 .btn-info:hover,
@@ -2539,7 +2519,6 @@ input[type="button"].btn-block {
   border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
   *background-color: #222222;
   /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
   filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
 }
 .btn-inverse:hover,
@@ -2614,7 +2593,6 @@ input[type="submit"].btn.btn-mini {
   display: inline-block;
   *display: inline;
   /* IE7 inline-block hack */
-
   *zoom: 1;
   font-size: 0;
   vertical-align: middle;
@@ -2790,7 +2768,6 @@ input[type="submit"].btn.btn-mini {
   display: inline-block;
   *display: inline;
   /* IE7 inline-block hack */
-
   *zoom: 1;
 }
 .btn-group-vertical > .btn {
@@ -3487,7 +3464,6 @@ input[type="submit"].btn.btn-mini {
   border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
   *background-color: #e5e5e5;
   /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
   filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
   -webkit-box-shadow: inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.075);
   -moz-box-shadow: inset 0 1px 0 rgba(255,255,255,.1), 0 1px 0 rgba(255,255,255,.075);
@@ -3721,7 +3697,6 @@ input[type="submit"].btn.btn-mini {
   border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
   *background-color: #040404;
   /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
   filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
 }
 .navbar-inverse .btn-navbar:hover,
@@ -3751,7 +3726,6 @@ input[type="submit"].btn.btn-mini {
   display: inline-block;
   *display: inline;
   /* IE7 inline-block hack */
-
   *zoom: 1;
   text-shadow: 0 1px 0 #ffffff;
 }
@@ -3769,7 +3743,6 @@ input[type="submit"].btn.btn-mini {
   display: inline-block;
   *display: inline;
   /* IE7 inline-block hack */
-
   *zoom: 1;
   margin-left: 0;
   margin-bottom: 0;
@@ -3970,7 +3943,6 @@ input[type="submit"].btn.btn-mini {
   border: 1px solid rgba(0, 0, 0, 0.3);
   *border: 1px solid #999;
   /* IE6-7 */
-
   -webkit-border-radius: 6px;
   -moz-border-radius: 6px;
   border-radius: 6px;
@@ -5828,7 +5800,6 @@ textarea {
   border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
   *background-color: #085871;
   /* Darken IE7 buttons by default so they stand out more given they won't have borders */
-
   filter: progid:DXImageTransform.Microsoft.gradient(enabled = false);
 }
 .control-custom.disabled .checkbox.btn:hover,
@@ -6107,7 +6078,6 @@ textarea {
   outline: 0;
   outline: thin dotted \9;
   /* IE6-9 */
-
   -webkit-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6);
   -moz-box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6);
   box-shadow: inset 0 1px 1px rgba(0,0,0,.075), 0 0 8px rgba(82,168,236,.6);
@@ -6130,6 +6100,43 @@ textarea {
   -moz-box-shadow: none;
   box-shadow: none;
 }
+.js .image-upload #field-image-upload {
+  cursor: pointer;
+  position: absolute;
+  z-index: 1;
+  opacity: 0;
+  filter: alpha(opacity=0);
+}
+.js .image-upload .controls {
+  position: relative;
+}
+.js .image-upload .btn {
+  position: relative;
+  top: 0;
+  margin-right: 10px;
+}
+.js .image-upload .btn.hover {
+  color: #333333;
+  text-decoration: none;
+  background-position: 0 -15px;
+  -webkit-transition: background-position 0.1s linear;
+  -moz-transition: background-position 0.1s linear;
+  -o-transition: background-position 0.1s linear;
+  transition: background-position 0.1s linear;
+}
+.js .image-upload .btn-remove-url {
+  position: absolute;
+  margin-right: 0;
+  top: 4px;
+  right: 5px;
+  padding: 0 4px;
+  -webkit-border-radius: 100px;
+  -moz-border-radius: 100px;
+  border-radius: 100px;
+}
+.js .image-upload .btn-remove-url .icon-remove {
+  margin-right: 0;
+}
 .dataset-item {
   border-bottom: 1px dotted #dddddd;
   padding-bottom: 20px;
@@ -6516,7 +6523,6 @@ textarea {
   margin-right: 5px;
   *display: inline;
   /* IE7 inline-block hack */
-
   *zoom: 1;
 }
 .actions li:last-of-type {
@@ -7925,9 +7931,9 @@ h4 small {
   -webkit-border-radius: 100px;
   -moz-border-radius: 100px;
   border-radius: 100px;
-  -webkit-box-shadow: inset 0 1px 2 x rgba(0, 0, 0, 0.2);
-  -moz-box-shadow: inset 0 1px 2 x rgba(0, 0, 0, 0.2);
-  box-shadow: inset 0 1px 2 x rgba(0, 0, 0, 0.2);
+  -webkit-box-shadow: inset 0 1px 2x rgba(0, 0, 0, 0.2);
+  -moz-box-shadow: inset 0 1px 2x rgba(0, 0, 0, 0.2);
+  box-shadow: inset 0 1px 2x rgba(0, 0, 0, 0.2);
 }
 .popover-followee .nav li a:hover i {
   background-color: #000;
@@ -8162,7 +8168,6 @@ iframe {
 .ie7 .control-custom .checkbox {
   *display: inline;
   /* IE7 inline-block hack */
-
   *zoom: 1;
 }
 .ie7 .stages {
@@ -8196,7 +8201,6 @@ iframe {
 .ie7 .masthead nav {
   *display: inline;
   /* IE7 inline-block hack */
-
   *zoom: 1;
 }
 .ie7 .masthead .header-image {
diff --git a/ckan/templates/macros/form.html b/ckan/templates/macros/form.html
index 5a6c00a6cb1..db93f046961 100644
--- a/ckan/templates/macros/form.html
+++ b/ckan/templates/macros/form.html
@@ -405,12 +405,12 @@
 #}
 {% macro image_upload(data, errors, field_url='image_url', field_upload='image_upload', field_clear='clear_upload', image_url=false, is_upload_enabled=false, placeholder=false) %}
   {% set placeholder = placeholder if placeholder else _('http://example.com/my-image.jpg') %}
-  {% set has_uploaded_data = data.image_url and not data.image_url.startswith('http') %}
-  {% set is_url = data.image_url and data.image_url.startswith('http') %}
+  {% set has_uploaded_data = data.get(field_url) and not data[field_url].startswith('http') %}
+  {% set is_url = data.get(field_url) and data[field_url].startswith('http') %}
 
   {% if is_upload_enabled %}<div class="image-upload" data-module="image-upload" data-module-is_url="{{ 'true' if is_url else 'false' }}" data-module-image_url="{{ image_url }}">{% endif %}
 
-  {{ input(field_url, label=_('Image URL'), id='field-image-url', placeholder=placeholder, value=data.image_url, error=errors.image_url, classes=['control-full']) }}
+  {{ input(field_url, label=_('Image URL'), id='field-image-url', placeholder=placeholder, value=data.get(field_url), error=errors.get(field_url), classes=['control-full']) }}
 
   {% if is_upload_enabled %}
     {{ input(field_upload, label=_('Image Upload'), id='field-image-upload', type='file', placeholder='', value='', error='', classes=['control-full']) }}

From 04d65785c97a6b8d44d70c8f6bdff4de50caf3b7 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Tue, 5 Nov 2013 16:03:23 +0000
Subject: [PATCH 82/87] [#960] Remove ckan.api_url completely on 2.2

---
 CHANGELOG.rst                       |  2 ++
 ckan/lib/app_globals.py             |  1 -
 ckan/public/base/javascript/main.js |  3 ---
 ckan/templates/base.html            |  2 +-
 doc/configuration.rst               | 18 ------------------
 doc/legacy-api.rst                  |  6 ------
 6 files changed, 3 insertions(+), 29 deletions(-)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index b97798a4522..17dfb5069c2 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -12,6 +12,8 @@ v2.2
 
 API changes and deprecations:
 
+
+* The `ckan.api_url` has been completely removed and it can no longer be used
 * The edit() and after_update() methods of IPackageController plugins are now
   called when updating a resource using the web frontend or the
   resource_update API action [#1052]
diff --git a/ckan/lib/app_globals.py b/ckan/lib/app_globals.py
index 38c777f7555..38138e87815 100644
--- a/ckan/lib/app_globals.py
+++ b/ckan/lib/app_globals.py
@@ -44,7 +44,6 @@
     'ckan.template_footer_end': {},
     'ckan.dumps_url': {},
     'ckan.dumps_format': {},
-    'ckan.api_url': {},
     'ofs.impl': {'name': 'ofs_impl'},
     'ckan.homepage_style': {'default': '1'},
 
diff --git a/ckan/public/base/javascript/main.js b/ckan/public/base/javascript/main.js
index 843dc47442d..c5f36ef7176 100644
--- a/ckan/public/base/javascript/main.js
+++ b/ckan/public/base/javascript/main.js
@@ -30,9 +30,6 @@ this.ckan = this.ckan || {};
     ckan.SITE_ROOT   = getRootFromData('siteRoot');
     ckan.LOCALE_ROOT = getRootFromData('localeRoot');
 
-    // Not used, kept for backwards compatibility
-    ckan.API_ROOT    = body.data('apiRoot') || ckan.SITE_ROOT;
-
     // Load the localisations before instantiating the modules.
     ckan.sandbox().client.getLocaleData(locale).done(function (data) {
       ckan.i18n.load(data);
diff --git a/ckan/templates/base.html b/ckan/templates/base.html
index 6012bf65928..b6290af0ad0 100644
--- a/ckan/templates/base.html
+++ b/ckan/templates/base.html
@@ -85,7 +85,7 @@
   </head>
 
   {# Allows custom attributes to be added to the <body> tag #}
-  <body{% block bodytag %} data-site-root="{{ h.url('/', locale='default', qualified=true) }}" data-locale-root="{{ h.url('/', qualified=true) }}" data-api-root="{{ g.api_url }}"{% endblock %}>
+  <body{% block bodytag %} data-site-root="{{ h.url('/', locale='default', qualified=true) }}" data-locale-root="{{ h.url('/', qualified=true) }}" {% endblock %}>
 
     {#
     The page block allows you to add content to the page. Most of the time it is
diff --git a/doc/configuration.rst b/doc/configuration.rst
index 0b623c5e656..c0768e148ab 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -130,24 +130,6 @@ site use this setting.
 
   This setting should not have a trailing / on the end.
 
-.. _ckan.api_url:
-
-ckan.api_url
-^^^^^^^^^^^^
-
-.. deprecated:: 2
-   No longer used.
-
-Example::
-
- ckan.api_url = http://scotdata.ckan.net/api
-
-Default value:  ``/api``
-
-The URL that resolves to the CKAN API part of the site. This is useful if the
-API is hosted on a different domain, for example when a third-party site uses
-the forms API.
-
 .. _apikey_header_name:
 
 apikey_header_name
diff --git a/doc/legacy-api.rst b/doc/legacy-api.rst
index b10841fc976..7ab990dc371 100644
--- a/doc/legacy-api.rst
+++ b/doc/legacy-api.rst
@@ -434,12 +434,6 @@ front-end javascript.
 All Util APIs are read-only. The response format is JSON. Javascript calls may
 want to use the JSONP formatting.
 
-.. Note::
-
-  Some CKAN deployments have the API deployed at a different domain to the main CKAN website. To make sure that the AJAX calls in the Web UI work, you'll need to configue the ckan.api_url. e.g.::
-
-    ckan.api_url = http://api.example.com/
-
 
 dataset autocomplete
 ````````````````````

From 4d0184ae6ee1af126c3103a4ac4c407c417fa824 Mon Sep 17 00:00:00 2001
From: kindly <kindly@gmail.com>
Date: Tue, 5 Nov 2013 17:27:27 +0000
Subject: [PATCH 83/87] [#1262] fix required field in wrong place

---
 ckan/templates/organization/snippets/organization_form.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ckan/templates/organization/snippets/organization_form.html b/ckan/templates/organization/snippets/organization_form.html
index 0b97b958d38..0d62e36cac1 100644
--- a/ckan/templates/organization/snippets/organization_form.html
+++ b/ckan/templates/organization/snippets/organization_form.html
@@ -71,8 +71,8 @@
   {% endblock %}
   #}
 
-  <div class="form-actions">
     {{ form.required_message() }}
+  <div class="form-actions">
       {% block delete_button %}
         {% if h.check_access('organization_delete', {'id': data.id})  %}
           {% set locale = h.dump_json({'content': _('Are you sure you want to delete this Organization? This will delete all the public and private datasets belonging to this organization.')}) %}

From ff7e38e56842d4e20b6846717e61625fbcbe47e2 Mon Sep 17 00:00:00 2001
From: John Glover <j@johnglover.net>
Date: Tue, 5 Nov 2013 18:38:22 +0100
Subject: [PATCH 84/87] [#1226] PEP8

---
 ckan/tests/functional/api/test_user.py | 38 ++++++++++++++------------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/ckan/tests/functional/api/test_user.py b/ckan/tests/functional/api/test_user.py
index e313e29710c..875e8a668ab 100644
--- a/ckan/tests/functional/api/test_user.py
+++ b/ckan/tests/functional/api/test_user.py
@@ -12,6 +12,7 @@
 import ckan.config.middleware
 from ckan.common import json
 
+
 class TestUserApi(ControllerTestCase):
     @classmethod
     def setup_class(cls):
@@ -68,8 +69,8 @@ def setup_class(cls):
         CreateTestData.create()
         cls._original_config = config.copy()
         new_authz.clear_auth_functions_cache()
-        wsgiapp = ckan.config.middleware.make_app(config['global_conf'],
-                **config)
+        wsgiapp = ckan.config.middleware.make_app(
+            config['global_conf'], **config)
         cls.app = paste.fixture.TestApp(wsgiapp)
         cls.sysadmin_user = model.User.get('testsysadmin')
         PylonsTestCase.setup_class()
@@ -89,9 +90,11 @@ def test_user_create_api_enabled_sysadmin(self):
             'email': 'testinganewuser@ckan.org',
             'password': 'random',
         }
-        res = self.app.post('/api/3/action/user_create', json.dumps(params),
-                extra_environ={'Authorization': str(self.sysadmin_user.apikey)},
-                expect_errors=True)
+        res = self.app.post(
+            '/api/3/action/user_create',
+            json.dumps(params),
+            extra_environ={'Authorization': str(self.sysadmin_user.apikey)},
+            expect_errors=True)
         res_dict = res.json
         assert res_dict['success'] is True
 
@@ -102,7 +105,7 @@ def test_user_create_api_disabled_anon(self):
             'password': 'random',
         }
         res = self.app.post('/api/3/action/user_create', json.dumps(params),
-                expect_errors=True)
+                            expect_errors=True)
         res_dict = res.json
         assert res_dict['success'] is False
 
@@ -118,8 +121,8 @@ def setup_class(cls):
         cls._original_config = config.copy()
         config['ckan.auth.create_user_via_api'] = True
         new_authz.clear_auth_functions_cache()
-        wsgiapp = ckan.config.middleware.make_app(config['global_conf'],
-                **config)
+        wsgiapp = ckan.config.middleware.make_app(
+            config['global_conf'], **config)
         cls.app = paste.fixture.TestApp(wsgiapp)
         PylonsTestCase.setup_class()
         cls.sysadmin_user = model.User.get('testsysadmin')
@@ -139,9 +142,10 @@ def test_user_create_api_enabled_sysadmin(self):
             'email': 'testinganewuser@ckan.org',
             'password': 'random',
         }
-        res = self.app.post('/api/3/action/user_create', json.dumps(params),
-                extra_environ={'Authorization':
-                    str(self.sysadmin_user.apikey)})
+        res = self.app.post(
+            '/api/3/action/user_create',
+            json.dumps(params),
+            extra_environ={'Authorization': str(self.sysadmin_user.apikey)})
         res_dict = res.json
         assert res_dict['success'] is True
 
@@ -167,8 +171,8 @@ def setup_class(cls):
         cls._original_config = config.copy()
         config['ckan.auth.create_user_via_web'] = False
         new_authz.clear_auth_functions_cache()
-        wsgiapp = ckan.config.middleware.make_app(config['global_conf'],
-                **config)
+        wsgiapp = ckan.config.middleware.make_app(
+            config['global_conf'], **config)
         cls.app = paste.fixture.TestApp(wsgiapp)
         cls.sysadmin_user = model.User.get('testsysadmin')
         PylonsTestCase.setup_class()
@@ -189,7 +193,7 @@ def test_user_create_api_disabled(self):
             'password': 'random',
         }
         res = self.app.post('/api/3/action/user_create', json.dumps(params),
-                expect_errors=True)
+                            expect_errors=True)
         res_dict = res.json
         assert res_dict['success'] is False
 
@@ -205,8 +209,8 @@ def setup_class(cls):
         cls._original_config = config.copy()
         config['ckan.auth.create_user_via_web'] = True
         new_authz.clear_auth_functions_cache()
-        wsgiapp = ckan.config.middleware.make_app(config['global_conf'],
-                **config)
+        wsgiapp = ckan.config.middleware.make_app(
+            config['global_conf'], **config)
         cls.app = paste.fixture.TestApp(wsgiapp)
         cls.sysadmin_user = model.User.get('testsysadmin')
         PylonsTestCase.setup_class()
@@ -227,7 +231,7 @@ def test_user_create_api_disabled(self):
             'password': 'random',
         }
         res = self.app.post('/api/3/action/user_create', json.dumps(params),
-                expect_errors=True)
+                            expect_errors=True)
         res_dict = res.json
         assert res_dict['success'] is False
 

From 85737e8e2d1e1b4184849f278c7e24e501e5e510 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Thu, 7 Nov 2013 11:41:45 +0000
Subject: [PATCH 85/87] [#1307] Minor docs correction

---
 ckan/logic/action/create.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py
index c7fb98be30a..4ead6b6cb9d 100644
--- a/ckan/logic/action/create.py
+++ b/ckan/logic/action/create.py
@@ -90,7 +90,7 @@ def package_create(context, data_dict):
     :param extras: the dataset's extras (optional), extras are arbitrary
         (key: value) metadata items that can be added to datasets, each extra
         dictionary should have keys ``'key'`` (a string), ``'value'`` (a
-        string), and optionally ``'deleted'``
+        string)
     :type extras: list of dataset extra dictionaries
     :param relationships_as_object: see ``package_relationship_create()`` for
         the format of relationship dictionaries (optional)

From c698cdd9565fdfcf7752f1b9d274d0574abab7b1 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Thu, 7 Nov 2013 18:02:52 +0000
Subject: [PATCH 86/87] [#1309] Fix placeholders in non-root locations

---
 ckan/templates/group/snippets/group_item.html               | 2 +-
 ckan/templates/organization/snippets/organization_item.html | 2 +-
 ckan/templates/related/snippets/related_item.html           | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ckan/templates/group/snippets/group_item.html b/ckan/templates/group/snippets/group_item.html
index 9e32ce86e6a..ccb072bfc44 100644
--- a/ckan/templates/group/snippets/group_item.html
+++ b/ckan/templates/group/snippets/group_item.html
@@ -14,7 +14,7 @@
 {% set url = h.url_for(group.type ~ '_read', action='read', id=group.name) %}
 <li class="media-item">
   {% block image %}
-    <img src="{{ group.image_url or '/base/images/placeholder-group.png' }}" alt="{{ group.name }}" class="media-image">
+    <img src="{{ group.image_url or h.url_for_static('/base/images/placeholder-group.png') }}" alt="{{ group.name }}" class="media-image">
   {% endblock %}
   {% block title %}
     <h3 class="media-heading">{{ group.display_name }}</h3>
diff --git a/ckan/templates/organization/snippets/organization_item.html b/ckan/templates/organization/snippets/organization_item.html
index 818d5cc338a..3020620b5f6 100644
--- a/ckan/templates/organization/snippets/organization_item.html
+++ b/ckan/templates/organization/snippets/organization_item.html
@@ -14,7 +14,7 @@
 {% set url = h.url_for(organization.type ~ '_read', action='read', id=organization.name) %}
 <li class="media-item">
   {% block image %}
-    <img src="{{ organization.image_url or '/base/images/placeholder-organization.png' }}" alt="{{ organization.name }}" class="media-image">
+    <img src="{{ organization.image_url or h.url_for_static('/base/images/placeholder-organization.png') }}" alt="{{ organization.name }}" class="media-image">
   {% endblock %}
   {% block title %}
     <h3 class="media-heading">{{ organization.display_name }}</h3>
diff --git a/ckan/templates/related/snippets/related_item.html b/ckan/templates/related/snippets/related_item.html
index 39231a7a217..df1c1300025 100644
--- a/ckan/templates/related/snippets/related_item.html
+++ b/ckan/templates/related/snippets/related_item.html
@@ -11,11 +11,11 @@
 
 #}
 {% set placeholder_map = {
-'application':'/base/images/placeholder-application.png'
+'application': h.url_for_static('/base/images/placeholder-application.png')
 } %}
 {% set tooltip = _('Go to {related_item_type}').format(related_item_type=related.type|replace('_', ' ')|title) %}
 <li class="related-item media-item" data-module="related-item">
-  <img src="{{ related.image_url or placeholder_map[related.type] or '/base/images/placeholder-image.png' }}" alt="{{ related.title }}" class="media-image">
+  <img src="{{ related.image_url or placeholder_map[related.type] or h.url_for_static('/base/images/placeholder-image.png') }}" alt="{{ related.title }}" class="media-image">
   <h3 class="media-heading">{{ related.title }}</h3>
   {% if related.description %}
     <div class="prose">{{ h.render_markdown(related.description) }}</div>

From 596c9504745acccdd7d7e21c0ba05ad4f3dd13e5 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 8 Nov 2013 10:52:02 +0000
Subject: [PATCH 87/87] Update CHANGELOG

---
 CHANGELOG.rst | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index b97798a4522..95a57a6f501 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -16,6 +16,22 @@ API changes and deprecations:
   called when updating a resource using the web frontend or the
   resource_update API action [#1052]
 
+v2.1.1 2013-11-8
+================
+
+Bug fixes:
+ * Fix errors on preview on non-root locations (#960)
+ * Fix place-holder images on non-root locations (#1309)
+ * Don't accept invalid URLs in resource proxy (#1106)
+ * Make sure came_from url is local (#1039)
+ * Fix logout redirect in non-root locations (#1025)
+ * Wrong auth checks for sysadmins on package_create (#1184)
+ * Don't return private datasets on package_list (#1295)
+ * Stop tracking failing when no lang/encoding headers (#1192)
+ * Fix for paster db clean command getting frozen
+ * Fix organization not set when editing a dataset (#1199)
+ * Fix PDF previews (#1194)
+ * Fix preview failing on private datastore resources (#1221)
 
 v2.1 2013-08-13
 ===============
@@ -96,6 +112,18 @@ Known issues:
  * Under certain authorization setups the frontend for the groups functionality
    may not work as expected (See #1176 #1175).
 
+v2.0.3 2013-11-8
+================
+
+Bug fixes:
+ * Fix errors on preview on non-root locations (#960)
+ * Don't accept invalid URLs in resource proxy (#1106)
+ * Make sure came_from url is local (#1039)
+ * Fix logout redirect in non-root locations (#1025)
+ * Don't return private datasets on package_list (#1295)
+ * Stop tracking failing when no lang/encoding headers (#1192)
+ * Fix for paster db clean command getting frozen
+
 
 v2.0.2 2013-08-13
 =================