[#1200] make resource data page for reuploading and make error messag…

…es better

ckan/config/routing.py

@@ -246,7 +246,7 @@ def make_map():
                   action='resource_read')
         m.connect('/dataset/{id}/resource_delete/{resource_id}',
                   action='resource_delete')
-        m.connect('/dataset/{id}/resource_edit/{resource_id}',
+        m.connect('resource_edit', '/dataset/{id}/resource_edit/{resource_id}',
                   action='resource_edit')
         m.connect('/dataset/{id}/resource/{resource_id}/download',
                   action='resource_download')

ckan/logic/schema.py

@@ -85,6 +85,7 @@ def default_resource_schema():
         'cache_last_updated': [ignore_missing, isodate],
         'webstore_last_updated': [ignore_missing, isodate],
         'tracking_summary': [ignore_missing],
+        'datastore_active': [ignore],
         '__extras': [ignore_missing, extras_unicode_convert, keep_extras],
     }
 

ckan/templates/package/new_resource.html

@@ -29,8 +29,16 @@ <h2 class="module-heading"><i class="icon-info-sign"></i> {{ _('What\'s a resour
 {% endblock %}
 
 {% block primary_content %}
-  {% if pkg_dict and pkg_dict.state != 'draft' %}
-    <header class="module-content page-header"></header>
+  {% set res = c.resource %}
+  {% if pkg_dict and pkg_dict.state != 'draft' and res %}
+    <header class="module-content page-header">
+      <ul class="nav nav-tabs">
+      {{ h.build_nav_icon('resource_edit', _('Edit Resource'), id=pkg_dict.id, resource_id=res.id) }}
+      {% if 'datapusher' in g.plugins %}
+          {{ h.build_nav_icon('resource_data', _('Resource Data'), id=pkg_dict.id, resource_id=res.id) }}
+      {% endif %}
+      </ul>
+    </header>
   {% endif %}
   {{ super() }}
 {% endblock %}

ckan/templates/package/resource_data.html

@@ -0,0 +1,45 @@
+{% extends "package/new_resource.html" %}
+
+{% set pkg_dict = c.pkg_dict %}
+{% set res = c.resource %}
+
+{% block subtitle %}{{ h.dataset_display_name(pkg_dict) }} - {{ h.resource_display_name(res) }}{% endblock %}
+
+{% block breadcrumb_content_selected %}{% endblock %}
+
+{% block breadcrumb_content %}
+  {{ super() }}
+  <li class="active"><a href="">{{ _('Edit') }}</a></li>
+{% endblock %}
+
+{% block content_action %}
+  {% link_for _('View resource'), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id, class_='btn', icon='eye-open' %}
+{% endblock %}
+
+{# logged_in is defined in new_resource.html #}
+{% block form %}
+
+   {% set action = h.url_for(controller='ckanext.datapusher.plugin:ResourceDataController', action='resource_data', id=pkg_dict.id, resource_id=res.id) %}
+
+   <form class="dataset-form dataset-resource-form form-horizontal" method="post" action="{{ action }}" >
+      <h3> {{ _('Last Upload Status:') }} {% if status.status %} {{ status.status.capitalize() }} {% else %} {{ _('Not Uploaded Yet') }}  {% endif %} </h3>
+      {% if status.error and status.error.message %}  
+        <h3>  {{ _('Upload Error:') }} {{ status.error.message }} </h3>
+      {% endif %} 
+      {% if status.status %}  
+        <h3>   {{ _('Last Updated: ') }} {{ h.time_ago_from_timestamp(status.last_updated) }} </h3>
+      {% endif %} 
+
+      <button class="btn" name="save" value="go-dataset" type="submit">{{ _('Upload Data') }}</button>
+      {% if status.status and status.task_info %}  
+        <h3>   {{ _('Upload Log ') }} </h3>
+        {{ h.debug_inspect(status.task_info.logs) }}
+      {% endif %} 
+      {{ status }}
+   </form>
+
+{% endblock %}
+
+{% block secondary_content %}
+  {% snippet 'package/snippets/info.html', pkg=pkg_dict, active=pkg_dict.id, action='resource_edit' %}
+{% endblock %}

ckan/templates/package/resource_read.html

@@ -27,9 +27,6 @@
           <ul>
             {% block resource_actions_inner %}
             {% if h.check_access('package_update', {'id':pkg.id }) %}
-              {% if 'datapusher' in g.plugins %}
-                <li>{% snippet 'snippets/datapusher_status.html', resource=res %}</li>
-              {% endif %}
               <li>{% link_for _('Edit'), controller='package', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn', icon='wrench' %}</li>
             {% endif %}
             {% if res.url %}

ckanext/datapusher/logic/action.py

@@ -51,6 +51,29 @@ def datapusher_submit(context, data_dict):
 
     user = p.toolkit.get_action('user_show')(context, {'id': context['user']})
 
+    task = {
+        'entity_id': res_id,
+        'entity_type': 'resource',
+        'task_type': 'datapusher',
+        'last_updated': str(datetime.datetime.now()),
+        'state': 'submitting',
+        'key': 'datapusher',
+        'value': '{}',
+        'error': '{}',
+    }
+    try:
+        task_id = p.toolkit.get_action('task_status_show')(context, {
+            'entity_id': res_id,
+            'task_type': 'datapusher',
+            'key': 'datapusher'
+        })['id']
+        task['id'] = task_id
+    except logic.NotFound:
+        pass
+
+    result = p.toolkit.get_action('task_status_update')(context, task)
+    task_id = result['id']
+
     try:
         r = requests.post(
             urlparse.urljoin(datapusher_url, 'job'),
@@ -69,36 +92,37 @@ def datapusher_submit(context, data_dict):
             }))
         r.raise_for_status()
     except requests.exceptions.ConnectionError, e:
-        raise p.toolkit.ValidationError({'datapusher': {
-            'message': 'Could not connect to DataPusher.',
-            'details': str(e)}})
+        error = {'message': 'Could not connect to DataPusher.',
+                 'details': str(e)}
+        task['error'] = json.dumps(error)
+        task['state'] = 'error'
+        task['last_updated'] = str(datetime.datetime.now()),
+        p.toolkit.get_action('task_status_update')(context, task)
+        raise p.toolkit.ValidationError(error)
+
     except requests.exceptions.HTTPError, e:
         m = 'An Error occurred while sending the job: {0}'.format(e.message)
         try:
             body = e.response.json()
         except ValueError:
             body = e.response.text
-        raise p.toolkit.ValidationError({'datapusher': {
-            'message': m,
-            'details': body,
-            'status_code': r.status_code}})
-
-    empty_task = {
-        'entity_id': res_id,
-        'entity_type': 'resource',
-        'task_type': 'datapusher',
-        'last_updated': str(datetime.datetime.now()),
-        'state': 'pending'
-    }
-
-    tasks = []
-    for (k, v) in [('job_id', r.json()['job_id']),
-                   ('job_key', r.json()['job_key'])]:
-        t = empty_task.copy()
-        t['key'] = k
-        t['value'] = v
-        tasks.append(t)
-    p.toolkit.get_action('task_status_update_many')(context, {'data': tasks})
+        error = {'message': m,
+                 'details': body,
+                 'status_code': r.status_code}
+        task['error'] = json.dumps(error)
+        task['state'] = 'error'
+        task['last_updated'] = str(datetime.datetime.now()),
+        p.toolkit.get_action('task_status_update')(context, task)
+        raise p.toolkit.ValidationError(error)
+
+    value = json.dumps(
+            {'job_id': r.json()['job_id'],
+             'job_key': r.json()['job_key']}
+            )
+    task['value'] = value
+    task['state'] = 'pending'
+    task['last_updated'] = str(datetime.datetime.now()),
+    p.toolkit.get_action('task_status_update')(context, task)
 
     return True
 
@@ -111,30 +135,20 @@ def datapusher_hook(context, data_dict):
     '''
 
     # TODO: use a schema to validate
-
     p.toolkit.check_access('datapusher_submit', context, data_dict)
 
     res_id = data_dict['metadata']['resource_id']
 
-    task_id = p.toolkit.get_action('task_status_show')(context, {
+    task = p.toolkit.get_action('task_status_show')(context, {
         'entity_id': res_id,
         'task_type': 'datapusher',
-        'key': 'job_id'
+        'key': 'datapusher'
     })
 
-    task_key = p.toolkit.get_action('task_status_show')(context, {
-        'entity_id': res_id,
-        'task_type': 'datapusher',
-        'key': 'job_key'
-    })
-
-    tasks = [task_id, task_key]
-
-    for task in tasks:
-        task['state'] = data_dict['status']
-        task['last_updated'] = str(datetime.datetime.now())
+    task['state'] = data_dict['status']
+    task['last_updated'] = str(datetime.datetime.now())
 
-    p.toolkit.get_action('task_status_update_many')(context, {'data': tasks})
+    p.toolkit.get_action('task_status_update')(context, task)
 
 
 def datapusher_status(context, data_dict):
@@ -151,28 +165,37 @@ def datapusher_status(context, data_dict):
         data_dict['resource_id'] = data_dict['id']
     res_id = _get_or_bust(data_dict, 'resource_id')
 
-    task_id = p.toolkit.get_action('task_status_show')(context, {
+    task = p.toolkit.get_action('task_status_show')(context, {
         'entity_id': res_id,
         'task_type': 'datapusher',
-        'key': 'job_id'
-    })
-
-    task_key = p.toolkit.get_action('task_status_show')(context, {
-        'entity_id': res_id,
-        'task_type': 'datapusher',
-        'key': 'job_key'
+        'key': 'datapusher'
     })
 
     datapusher_url = pylons.config.get('ckan.datapusher.url')
     if not datapusher_url:
         raise p.toolkit.ValidationError(
             {'configuration': ['DataPusher not configured.']})
 
-    url = urlparse.urljoin(datapusher_url, 'job' + '/' + task_id['value'])
+    value = json.loads(task['value'])
+    job_key = value.get('job_key')
+    job_id = value.get('job_id')
+    url = None
+    job_detail = None
+    if job_id:
+        url = urlparse.urljoin(datapusher_url, 'job' + '/' + job_id)
+        try:
+            r = requests.get(url, headers={'Content-Type': 'application/json',
+                                           'Authorization': job_key})
+            job_detail = r.json()
+        except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e:
+            job_detail = {'error': 'cannot connect to datapusher'}
+
     return {
-        'status': task_id['state'],
-        'job_id': task_id['value'],
+        'status': task['state'],
+        'job_id': job_id,
         'job_url': url,
-        'last_updated': task_id['last_updated'],
-        'job_key': task_key['value']
+        'last_updated': task['last_updated'],
+        'job_key': job_key,
+        'task_info': job_detail,
+        'error': json.loads(task['error'])
     }

ckanext/datapusher/plugin.py

@@ -1,11 +1,14 @@
 import logging
 
 import ckan.plugins as p
+import ckan.lib.base as base
+import ckan.lib.helpers as core_helpers
 import ckanext.datapusher.logic.action as action
 import ckanext.datapusher.logic.auth as auth
 import ckanext.datapusher.helpers as helpers
 import ckan.logic as logic
 import ckan.model as model
+from ckan.common import c, _, request
 
 log = logging.getLogger(__name__)
 _get_or_bust = logic.get_or_bust
@@ -16,6 +19,48 @@
 class DatastoreException(Exception):
     pass
 
+class ResourceDataController(base.BaseController):
+
+    def resource_data(self, id, resource_id):
+
+        if request.method == 'POST':
+            result = request.POST
+            try:
+                c.pkg_dict = p.toolkit.get_action('datapusher_submit')(
+                    None, {'resource_id': resource_id}
+                )
+            except logic.ValidationError:
+                pass
+
+            base.redirect(core_helpers.url_for(
+                 controller='ckanext.datapusher.plugin:ResourceDataController',
+                 action='resource_data',
+                 id=id,
+                 resource_id=resource_id)
+            )
+
+        try:
+            c.pkg_dict = p.toolkit.get_action('package_show')(
+                None, {'id': id}
+            )
+            c.resource = p.toolkit.get_action('resource_show')(
+                None, {'id': resource_id}
+            )
+        except logic.NotFound:
+            base.abort(404, _('Resource not found'))
+        except logic.NotAuthorized:
+            base.abort(401, _('Unauthorized to edit this resource'))
+
+        try:
+            datapusher_status = p.toolkit.get_action('datapusher_status')(
+                None, {'resource_id': resource_id}
+            )
+        except logic.NotFound:
+            datapusher_status = {}
+
+        return base.render('package/resource_data.html',
+                            extra_vars={'status': datapusher_status})
+
 
 class DatapusherPlugin(p.SingletonPlugin):
     p.implements(p.IConfigurable, inherit=True)
@@ -24,6 +69,7 @@ class DatapusherPlugin(p.SingletonPlugin):
     p.implements(p.IResourceUrlChange)
     p.implements(p.IDomainObjectModification, inherit=True)
     p.implements(p.ITemplateHelpers)
+    p.implements(p.IRoutes, inherit=True)
 
     legacy_mode = False
     resource_show_action = None
@@ -39,6 +85,11 @@ def configure(self, config):
             raise Exception(
                 'Config option `ckan.datapusher.url` has to be set.')
 
+        self.datapusher_secret_key = config.get('ckan.datapusher.secret_key', '')
+        if not datapusher_url:
+            raise Exception(
+                'Config option `ckan.datapusher.secret_key` has to be set.')
+
     def notify(self, entity, operation=None):
         if isinstance(entity, model.Resource):
             if (operation == model.domain_object.DomainObjectOperation.new
@@ -63,6 +114,11 @@ def notify(self, entity, operation=None):
                         log.critical(e)
                         pass
 
+    def before_map(self, m):
+        m.connect('resource_data', '/dataset/{id}/resource_data/{resource_id}',
+                  controller='ckanext.datapusher.plugin:ResourceDataController',
+                  action='resource_data')
+        return m
 
     def get_actions(self):
         return {'datapusher_submit': action.datapusher_submit,

ckanext/datastore/controller.py

@@ -42,3 +42,4 @@ def dump(self, resource_id):
         for record in result['records']:
             wr.writerow([record[column] for column in header])
         return f.getvalue()
+