diff --git a/ckan/config/routing.py b/ckan/config/routing.py
index 0fbb02bb886..6b41d8a66f2 100644
--- a/ckan/config/routing.py
+++ b/ckan/config/routing.py
@@ -246,7 +246,7 @@ def make_map():
action='resource_read')
m.connect('/dataset/{id}/resource_delete/{resource_id}',
action='resource_delete')
- m.connect('/dataset/{id}/resource_edit/{resource_id}',
+ m.connect('resource_edit', '/dataset/{id}/resource_edit/{resource_id}',
action='resource_edit')
m.connect('/dataset/{id}/resource/{resource_id}/download',
action='resource_download')
diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py
index f8fe8df7fbd..8058e10aea0 100644
--- a/ckan/logic/schema.py
+++ b/ckan/logic/schema.py
@@ -85,6 +85,7 @@ def default_resource_schema():
'cache_last_updated': [ignore_missing, isodate],
'webstore_last_updated': [ignore_missing, isodate],
'tracking_summary': [ignore_missing],
+ 'datastore_active': [ignore],
'__extras': [ignore_missing, extras_unicode_convert, keep_extras],
}
diff --git a/ckan/templates/package/new_resource.html b/ckan/templates/package/new_resource.html
index 40d8ed97c9e..7eedbad761f 100644
--- a/ckan/templates/package/new_resource.html
+++ b/ckan/templates/package/new_resource.html
@@ -29,8 +29,16 @@
{{ _('What\'s a resour
{% endblock %}
{% block primary_content %}
- {% if pkg_dict and pkg_dict.state != 'draft' %}
-
+ {% set res = c.resource %}
+ {% if pkg_dict and pkg_dict.state != 'draft' and res %}
+
{% endif %}
{{ super() }}
{% endblock %}
diff --git a/ckan/templates/package/resource_data.html b/ckan/templates/package/resource_data.html
new file mode 100644
index 00000000000..30c9fe59e98
--- /dev/null
+++ b/ckan/templates/package/resource_data.html
@@ -0,0 +1,45 @@
+{% extends "package/new_resource.html" %}
+
+{% set pkg_dict = c.pkg_dict %}
+{% set res = c.resource %}
+
+{% block subtitle %}{{ h.dataset_display_name(pkg_dict) }} - {{ h.resource_display_name(res) }}{% endblock %}
+
+{% block breadcrumb_content_selected %}{% endblock %}
+
+{% block breadcrumb_content %}
+ {{ super() }}
+
{{ _('Edit') }}
+{% endblock %}
+
+{% block content_action %}
+ {% link_for _('View resource'), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id, class_='btn', icon='eye-open' %}
+{% endblock %}
+
+{# logged_in is defined in new_resource.html #}
+{% block form %}
+
+ {% set action = h.url_for(controller='ckanext.datapusher.plugin:ResourceDataController', action='resource_data', id=pkg_dict.id, resource_id=res.id) %}
+
+
+
+{% endblock %}
+
+{% block secondary_content %}
+ {% snippet 'package/snippets/info.html', pkg=pkg_dict, active=pkg_dict.id, action='resource_edit' %}
+{% endblock %}
diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html
index f914780d179..46406746a6f 100644
--- a/ckan/templates/package/resource_read.html
+++ b/ckan/templates/package/resource_read.html
@@ -27,9 +27,6 @@
{% block resource_actions_inner %}
{% if h.check_access('package_update', {'id':pkg.id }) %}
- {% if 'datapusher' in g.plugins %}
- - {% snippet 'snippets/datapusher_status.html', resource=res %}
- {% endif %}
- {% link_for _('Edit'), controller='package', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn', icon='wrench' %}
{% endif %}
{% if res.url %}
diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py
index 3201d35572a..653dd44e0b5 100644
--- a/ckanext/datapusher/logic/action.py
+++ b/ckanext/datapusher/logic/action.py
@@ -51,6 +51,29 @@ def datapusher_submit(context, data_dict):
user = p.toolkit.get_action('user_show')(context, {'id': context['user']})
+ task = {
+ 'entity_id': res_id,
+ 'entity_type': 'resource',
+ 'task_type': 'datapusher',
+ 'last_updated': str(datetime.datetime.now()),
+ 'state': 'submitting',
+ 'key': 'datapusher',
+ 'value': '{}',
+ 'error': '{}',
+ }
+ try:
+ task_id = p.toolkit.get_action('task_status_show')(context, {
+ 'entity_id': res_id,
+ 'task_type': 'datapusher',
+ 'key': 'datapusher'
+ })['id']
+ task['id'] = task_id
+ except logic.NotFound:
+ pass
+
+ result = p.toolkit.get_action('task_status_update')(context, task)
+ task_id = result['id']
+
try:
r = requests.post(
urlparse.urljoin(datapusher_url, 'job'),
@@ -69,36 +92,37 @@ def datapusher_submit(context, data_dict):
}))
r.raise_for_status()
except requests.exceptions.ConnectionError, e:
- raise p.toolkit.ValidationError({'datapusher': {
- 'message': 'Could not connect to DataPusher.',
- 'details': str(e)}})
+ error = {'message': 'Could not connect to DataPusher.',
+ 'details': str(e)}
+ task['error'] = json.dumps(error)
+ task['state'] = 'error'
+ task['last_updated'] = str(datetime.datetime.now()),
+ p.toolkit.get_action('task_status_update')(context, task)
+ raise p.toolkit.ValidationError(error)
+
except requests.exceptions.HTTPError, e:
m = 'An Error occurred while sending the job: {0}'.format(e.message)
try:
body = e.response.json()
except ValueError:
body = e.response.text
- raise p.toolkit.ValidationError({'datapusher': {
- 'message': m,
- 'details': body,
- 'status_code': r.status_code}})
-
- empty_task = {
- 'entity_id': res_id,
- 'entity_type': 'resource',
- 'task_type': 'datapusher',
- 'last_updated': str(datetime.datetime.now()),
- 'state': 'pending'
- }
-
- tasks = []
- for (k, v) in [('job_id', r.json()['job_id']),
- ('job_key', r.json()['job_key'])]:
- t = empty_task.copy()
- t['key'] = k
- t['value'] = v
- tasks.append(t)
- p.toolkit.get_action('task_status_update_many')(context, {'data': tasks})
+ error = {'message': m,
+ 'details': body,
+ 'status_code': r.status_code}
+ task['error'] = json.dumps(error)
+ task['state'] = 'error'
+ task['last_updated'] = str(datetime.datetime.now()),
+ p.toolkit.get_action('task_status_update')(context, task)
+ raise p.toolkit.ValidationError(error)
+
+ value = json.dumps(
+ {'job_id': r.json()['job_id'],
+ 'job_key': r.json()['job_key']}
+ )
+ task['value'] = value
+ task['state'] = 'pending'
+ task['last_updated'] = str(datetime.datetime.now()),
+ p.toolkit.get_action('task_status_update')(context, task)
return True
@@ -111,30 +135,20 @@ def datapusher_hook(context, data_dict):
'''
# TODO: use a schema to validate
-
p.toolkit.check_access('datapusher_submit', context, data_dict)
res_id = data_dict['metadata']['resource_id']
- task_id = p.toolkit.get_action('task_status_show')(context, {
+ task = p.toolkit.get_action('task_status_show')(context, {
'entity_id': res_id,
'task_type': 'datapusher',
- 'key': 'job_id'
+ 'key': 'datapusher'
})
- task_key = p.toolkit.get_action('task_status_show')(context, {
- 'entity_id': res_id,
- 'task_type': 'datapusher',
- 'key': 'job_key'
- })
-
- tasks = [task_id, task_key]
-
- for task in tasks:
- task['state'] = data_dict['status']
- task['last_updated'] = str(datetime.datetime.now())
+ task['state'] = data_dict['status']
+ task['last_updated'] = str(datetime.datetime.now())
- p.toolkit.get_action('task_status_update_many')(context, {'data': tasks})
+ p.toolkit.get_action('task_status_update')(context, task)
def datapusher_status(context, data_dict):
@@ -151,16 +165,10 @@ def datapusher_status(context, data_dict):
data_dict['resource_id'] = data_dict['id']
res_id = _get_or_bust(data_dict, 'resource_id')
- task_id = p.toolkit.get_action('task_status_show')(context, {
+ task = p.toolkit.get_action('task_status_show')(context, {
'entity_id': res_id,
'task_type': 'datapusher',
- 'key': 'job_id'
- })
-
- task_key = p.toolkit.get_action('task_status_show')(context, {
- 'entity_id': res_id,
- 'task_type': 'datapusher',
- 'key': 'job_key'
+ 'key': 'datapusher'
})
datapusher_url = pylons.config.get('ckan.datapusher.url')
@@ -168,11 +176,26 @@ def datapusher_status(context, data_dict):
raise p.toolkit.ValidationError(
{'configuration': ['DataPusher not configured.']})
- url = urlparse.urljoin(datapusher_url, 'job' + '/' + task_id['value'])
+ value = json.loads(task['value'])
+ job_key = value.get('job_key')
+ job_id = value.get('job_id')
+ url = None
+ job_detail = None
+ if job_id:
+ url = urlparse.urljoin(datapusher_url, 'job' + '/' + job_id)
+ try:
+ r = requests.get(url, headers={'Content-Type': 'application/json',
+ 'Authorization': job_key})
+ job_detail = r.json()
+ except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError), e:
+ job_detail = {'error': 'cannot connect to datapusher'}
+
return {
- 'status': task_id['state'],
- 'job_id': task_id['value'],
+ 'status': task['state'],
+ 'job_id': job_id,
'job_url': url,
- 'last_updated': task_id['last_updated'],
- 'job_key': task_key['value']
+ 'last_updated': task['last_updated'],
+ 'job_key': job_key,
+ 'task_info': job_detail,
+ 'error': json.loads(task['error'])
}
diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py
index 04a27db435b..a545323a7dd 100644
--- a/ckanext/datapusher/plugin.py
+++ b/ckanext/datapusher/plugin.py
@@ -1,11 +1,14 @@
import logging
import ckan.plugins as p
+import ckan.lib.base as base
+import ckan.lib.helpers as core_helpers
import ckanext.datapusher.logic.action as action
import ckanext.datapusher.logic.auth as auth
import ckanext.datapusher.helpers as helpers
import ckan.logic as logic
import ckan.model as model
+from ckan.common import c, _, request
log = logging.getLogger(__name__)
_get_or_bust = logic.get_or_bust
@@ -16,6 +19,48 @@
class DatastoreException(Exception):
pass
+class ResourceDataController(base.BaseController):
+
+ def resource_data(self, id, resource_id):
+
+ if request.method == 'POST':
+ result = request.POST
+ try:
+ c.pkg_dict = p.toolkit.get_action('datapusher_submit')(
+ None, {'resource_id': resource_id}
+ )
+ except logic.ValidationError:
+ pass
+
+ base.redirect(core_helpers.url_for(
+ controller='ckanext.datapusher.plugin:ResourceDataController',
+ action='resource_data',
+ id=id,
+ resource_id=resource_id)
+ )
+
+ try:
+ c.pkg_dict = p.toolkit.get_action('package_show')(
+ None, {'id': id}
+ )
+ c.resource = p.toolkit.get_action('resource_show')(
+ None, {'id': resource_id}
+ )
+ except logic.NotFound:
+ base.abort(404, _('Resource not found'))
+ except logic.NotAuthorized:
+ base.abort(401, _('Unauthorized to edit this resource'))
+
+ try:
+ datapusher_status = p.toolkit.get_action('datapusher_status')(
+ None, {'resource_id': resource_id}
+ )
+ except logic.NotFound:
+ datapusher_status = {}
+
+ return base.render('package/resource_data.html',
+ extra_vars={'status': datapusher_status})
+
class DatapusherPlugin(p.SingletonPlugin):
p.implements(p.IConfigurable, inherit=True)
@@ -24,6 +69,7 @@ class DatapusherPlugin(p.SingletonPlugin):
p.implements(p.IResourceUrlChange)
p.implements(p.IDomainObjectModification, inherit=True)
p.implements(p.ITemplateHelpers)
+ p.implements(p.IRoutes, inherit=True)
legacy_mode = False
resource_show_action = None
@@ -39,6 +85,11 @@ def configure(self, config):
raise Exception(
'Config option `ckan.datapusher.url` has to be set.')
+ self.datapusher_secret_key = config.get('ckan.datapusher.secret_key', '')
+ if not datapusher_url:
+ raise Exception(
+ 'Config option `ckan.datapusher.secret_key` has to be set.')
+
def notify(self, entity, operation=None):
if isinstance(entity, model.Resource):
if (operation == model.domain_object.DomainObjectOperation.new
@@ -63,6 +114,11 @@ def notify(self, entity, operation=None):
log.critical(e)
pass
+ def before_map(self, m):
+ m.connect('resource_data', '/dataset/{id}/resource_data/{resource_id}',
+ controller='ckanext.datapusher.plugin:ResourceDataController',
+ action='resource_data')
+ return m
def get_actions(self):
return {'datapusher_submit': action.datapusher_submit,
diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py
index 3bb5def4e73..f4a14a352c6 100644
--- a/ckanext/datastore/controller.py
+++ b/ckanext/datastore/controller.py
@@ -42,3 +42,4 @@ def dump(self, resource_id):
for record in result['records']:
wr.writerow([record[column] for column in header])
return f.getvalue()
+