Merge pull request #3414 from ckan/3414-datastore-info

datastore Data Dictionary as postgresql comments

ckan/templates/package/resource_edit_base.html

@@ -22,9 +22,7 @@
 
 {% block content_primary_nav %}
   {{ h.build_nav_icon('resource_edit', _('Edit resource'), id=pkg.name, resource_id=res.id) }}
-  {% if 'datapusher' in g.plugins %}
-    {{ h.build_nav_icon('resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }}
-  {% endif %}
+  {% block inner_primary_nav %}{% endblock %}
   {{ h.build_nav_icon('views', _('Views'), id=pkg.name, resource_id=res.id) }}
 {% endblock %}
 

ckanext/datapusher/plugin.py

@@ -67,11 +67,12 @@ def resource_data(self, id, resource_id):
         except logic.NotAuthorized:
             base.abort(403, _('Not authorized to see this page'))
 
-        return base.render('package/resource_data.html',
+        return base.render('datapusher/resource_data.html',
                            extra_vars={'status': datapusher_status})
 
 
 class DatapusherPlugin(p.SingletonPlugin):
+    p.implements(p.IConfigurer, inherit=True)
     p.implements(p.IConfigurable, inherit=True)
     p.implements(p.IActions)
     p.implements(p.IAuthFunctions)
@@ -83,6 +84,9 @@ class DatapusherPlugin(p.SingletonPlugin):
     legacy_mode = False
     resource_show_action = None
 
+    def update_config(self, config):
+        p.toolkit.add_template_directory(config, 'templates')
+
     def configure(self, config):
         self.config = config
 

ckanext/datapusher/templates/package/resource_edit_base.html

@@ -0,0 +1,6 @@
+{% ckan_extends %}
+
+{% block inner_primary_nav %}
+  {{ super() }}
+  {{ h.build_nav_icon('resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }}
+{% endblock %}

ckanext/datastore/controller.py

@@ -1,19 +1,24 @@
 # encoding: utf-8
 
 import StringIO
+import md5
 
 import pylons
 
 from ckan.plugins.toolkit import (
     Invalid,
     ObjectNotFound,
+    NotAuthorized,
     get_action,
     get_validator,
     _,
     request,
     response,
     BaseController,
     abort,
+    render,
+    c,
+    h,
 )
 from ckanext.datastore.writer import (
     csv_writer,
@@ -84,3 +89,42 @@ def result_page(offset, limit):
                     limit -= PAGINATE_BY
 
                 result = result_page(offset, limit)
+
+    def dictionary(self, id, resource_id):
+        u'''data dictionary view: show/edit field labels and descriptions'''
+
+        try:
+            # resource_edit_base template uses these
+            c.pkg_dict = get_action('package_show')(
+                None, {'id': id})
+            c.resource = get_action('resource_show')(
+                None, {'id': resource_id})
+            rec = get_action('datastore_search')(None, {
+                'resource_id': resource_id,
+                'limit': 0})
+        except (ObjectNotFound, NotAuthorized):
+            abort(404, _('Resource not found'))
+
+        fields = [f for f in rec['fields'] if not f['id'].startswith('_')]
+
+        if request.method == 'POST':
+            get_action('datastore_create')(None, {
+                'resource_id': resource_id,
+                'force': True,
+                'fields': [{
+                    'id': f['id'],
+                    'type': f['type'],
+                    'info': {
+                        'label': request.POST.get('f{0}label'.format(i)),
+                        'notes': request.POST.get('f{0}notes'.format(i)),
+                        }} for i, f in enumerate(fields, 1)]})
+
+            h.redirect_to(
+                controller='ckanext.datastore.controller:DatastoreController',
+                action='dictionary',
+                id=id,
+                resource_id=resource_id)
+
+        return render(
+            'datastore/dictionary.html',
+            extra_vars={'fields': fields})

ckanext/datastore/db.py

@@ -226,6 +226,24 @@ def _guess_type(field):
     return 'text'
 
 
+def _get_field_info(connection, resource_id):
+    u'''return a dictionary mapping column names to their info data,
+    when present'''
+    qtext = sqlalchemy.text(u'''
+        select pa.attname as name, pd.description as info
+        from pg_class pc, pg_attribute pa, pg_description pd
+        where pa.attrelid = pc.oid and pd.objoid = pc.oid
+            and pd.objsubid = pa.attnum and pc.relname = :res_id
+            and pa.attnum > 0
+    ''')
+    try:
+        return dict(
+            (n, json.loads(v)) for (n, v) in
+            connection.execute(qtext, res_id=resource_id).fetchall())
+    except ValueError:  # don't die on non-json comments
+        return {}
+
+
 def _get_fields(context, data_dict):
     fields = []
     all_fields = context['connection'].execute(
@@ -339,15 +357,26 @@ def create_table(context, data_dict):
                 })
 
     fields = datastore_fields + supplied_fields + extra_fields
-    sql_fields = u", ".join([u'"{0}" {1}'.format(
-        f['id'], f['type']) for f in fields])
+    sql_fields = u", ".join([u'{0} {1}'.format(
+        datastore_helpers.identifier(f['id']), f['type']) for f in fields])
 
-    sql_string = u'CREATE TABLE "{0}" ({1});'.format(
-        data_dict['resource_id'],
+    sql_string = u'CREATE TABLE {0} ({1});'.format(
+        datastore_helpers.identifier(data_dict['resource_id']),
         sql_fields
     )
 
-    context['connection'].execute(sql_string.replace('%', '%%'))
+    info_sql = []
+    for f in supplied_fields:
+        info = f.get(u'info')
+        if isinstance(info, dict):
+            info_sql.append(u'COMMENT ON COLUMN {0}.{1} is {2}'.format(
+                datastore_helpers.identifier(data_dict['resource_id']),
+                datastore_helpers.identifier(f['id']),
+                datastore_helpers.literal_string(
+                    json.dumps(info, ensure_ascii=False))))
+
+    context['connection'].execute(
+        (sql_string + u';'.join(info_sql)).replace(u'%', u'%%'))
 
 
 def _get_aliases(context, data_dict):
@@ -607,12 +636,29 @@ def alter_table(context, data_dict):
                     'type': _guess_type(records[0][field_id])
                 })
 
-    for field in new_fields:
-        sql = 'ALTER TABLE "{0}" ADD "{1}" {2}'.format(
-            data_dict['resource_id'],
-            field['id'],
-            field['type'])
-        context['connection'].execute(sql.replace('%', '%%'))
+    alter_sql = []
+    for f in new_fields:
+        alter_sql.append(u'ALTER TABLE {0} ADD {1} {2};'.format(
+            datastore_helpers.identifier(data_dict['resource_id']),
+            datastore_helpers.identifier(f['id']),
+            f['type']))
+
+    for f in supplied_fields:
+        if u'info' in f:
+            info = f.get(u'info')
+            if isinstance(info, dict):
+                info_sql = datastore_helpers.literal_string(
+                    json.dumps(info, ensure_ascii=False))
+            else:
+                info_sql = 'NULL'
+            alter_sql.append(u'COMMENT ON COLUMN {0}.{1} is {2}'.format(
+                datastore_helpers.identifier(data_dict['resource_id']),
+                datastore_helpers.identifier(f['id']),
+                info_sql))
+
+    if alter_sql:
+        context['connection'].execute(
+            u';'.join(alter_sql).replace(u'%', u'%%'))
 
 
 def insert_data(context, data_dict):
@@ -993,7 +1039,8 @@ def search_data(context, data_dict):
     results = _execute_single_statement(context, sql_string, where_values)
 
     _insert_links(data_dict, limit, offset)
-    return format_results(context, results, data_dict)
+    return format_results(context, results, data_dict, _get_field_info(
+        context['connection'], data_dict['resource_id']))
 
 
 def _execute_single_statement(context, sql_string, where_values):
@@ -1007,13 +1054,16 @@ def _execute_single_statement(context, sql_string, where_values):
     return results
 
 
-def format_results(context, results, data_dict):
+def format_results(context, results, data_dict, field_info=None):
     result_fields = []
     for field in results.cursor.description:
-        result_fields.append({
+        f = {
             'id': field[0].decode('utf-8'),
             'type': _get_type(context, field[1])
-        })
+        }
+        if field_info and f['id'] in field_info:
+            f['info'] = field_info[f['id']]
+        result_fields.append(f)
     if len(result_fields) and result_fields[-1]['id'] == '_full_count':
         result_fields.pop()  # remove _full_count
 

ckanext/datastore/helpers.py

@@ -6,6 +6,8 @@
 import paste.deploy.converters as converters
 import sqlparse
 
+from ckan.plugins.toolkit import get_action, ObjectNotFound, NotAuthorized
+
 log = logging.getLogger(__name__)
 
 
@@ -107,3 +109,16 @@ def identifier(s):
     Return s as a quoted postgres identifier
     """
     return u'"' + s.replace(u'"', u'""').replace(u'\0', '') + u'"'
+
+
+def datastore_dictionary(resource_id):
+    """
+    Return the data dictionary info for a resource
+    """
+    try:
+        return [
+            f for f in get_action('datastore_search')(
+                None, {u'resource_id': resource_id, u'limit': 0})['fields']
+            if not f['id'].startswith(u'_')]
+    except (ObjectNotFound, NotAuthorized):
+        return []

ckanext/datastore/logic/schema.py

@@ -101,7 +101,8 @@ def datastore_create_schema():
         'aliases': [ignore_missing, list_of_strings_or_string],
         'fields': {
             'id': [not_empty, unicode],
-            'type': [ignore_missing]
+            'type': [ignore_missing],
+            'info': [ignore_missing],
         },
         'primary_key': [ignore_missing, list_of_strings_or_string],
         'indexes': [ignore_missing, list_of_strings_or_string],

ckanext/datastore/plugin.py

@@ -56,6 +56,7 @@ class DatastorePlugin(p.SingletonPlugin):
     p.implements(p.IDomainObjectModification, inherit=True)
     p.implements(p.IRoutes, inherit=True)
     p.implements(p.IResourceController, inherit=True)
+    p.implements(p.ITemplateHelpers)
     p.implements(interfaces.IDatastore, inherit=True)
 
     legacy_mode = False
@@ -247,9 +248,14 @@ def get_auth_functions(self):
                 'datastore_change_permissions': auth.datastore_change_permissions}
 
     def before_map(self, m):
-        m.connect('/datastore/dump/{resource_id}',
-                  controller='ckanext.datastore.controller:DatastoreController',
-                  action='dump')
+        m.connect(
+            '/datastore/dump/{resource_id}',
+            controller='ckanext.datastore.controller:DatastoreController',
+            action='dump')
+        m.connect(
+            'resource_dictionary', '/dataset/{id}/dictionary/{resource_id}',
+            controller='ckanext.datastore.controller:DatastoreController',
+            action='dictionary', ckan_icon='book')
         return m
 
     # IResourceController
@@ -532,3 +538,7 @@ def _ts_rank_alias(self, field=None):
         if field:
             rank_alias += u' ' + field
         return u'"{0}"'.format(rank_alias)
+
+    def get_helpers(self):
+        return {
+            'datastore_dictionary': datastore_helpers.datastore_dictionary}

ckanext/datastore/templates/datastore/dictionary.html

@@ -0,0 +1,29 @@
+{% extends "package/resource_edit_base.html" %}
+
+{% import 'macros/form.html' as form %}
+
+{% block subtitle %}{{ h.dataset_display_name(pkg) }} - {{ h.resource_display_name(res) }}{% endblock %}
+
+{% block primary_content_inner %}
+
+  {% set action = h.url_for(
+    controller='ckanext.datastore.controller:DatastoreController',
+    action='dictionary', id=pkg.name, resource_id=res.id) %}
+
+  <form method="post" action="{{ action }}" >
+    {% block dictionary_form %}
+      {% for f in fields %}
+        <h3>{{ _( "Field {num}.").format(num=loop.index) }} {{ f.id }} ({{ f.type }})</h3>
+        {{ form.input('f' ~ loop.index ~ 'label',
+          label=_('Label'), id='field-f' ~ loop.index ~ 'label',
+          value=f.get('info', {}).get('label', ''), classes=['control-full']) }}
+        {{ form.markdown('f' ~ loop.index ~ 'notes',
+          label=_('Description'), id='field-d' ~ loop.index ~ 'notes',
+          value=f.get('info', {}).get('notes', '')) }}
+      {% endfor %}
+    {% endblock %}
+    <button class="btn btn-primary" name="save" type="submit">
+      <i class="fa fa-book"></i> {{ _('Save') }}
+    </button>
+  </form>
+{% endblock %}

ckanext/datastore/templates/package/resource_edit_base.html

@@ -0,0 +1,8 @@
+{% ckan_extends %}
+
+{% block inner_primary_nav %}
+  {{ super() }}
+  {% if res.datastore_active %}
+    {{ h.build_nav_icon('resource_dictionary', _('Data Dictionary'), id=pkg.name, resource_id=res.id) }}
+  {% endif %}
+{% endblock %}

ckanext/datastore/templates/package/resource_read.html

@@ -2,5 +2,36 @@
 
 {% block resource_actions_inner %}
   {{ super() }}
-  <li>{% snippet 'package/snippets/data_api_button.html', resource=res,  datastore_root_url=c.datastore_api %}</li>
-{% endblock %}
+  {% if res.datastore_active %}
+    <li>{% snippet 'package/snippets/data_api_button.html', resource=res,  datastore_root_url=c.datastore_api %}</li>
+  {% endif %}
+{% endblock %}
+
+{% block resource_additional_information_inner %}
+  {% if res.datastore_active %}
+    {% set ddict=h.datastore_dictionary(res.id) %}
+    <div class="module-content">
+      <h2>{{ _('Data Dictionary') }}</h2>
+      <table class="table table-striped table-bordered table-condensed" data-module="table-toggle-more">
+        <thead>
+          <tr>
+            <th scope="col">{{ _('Column') }}</th>
+            <th scope="col">{{ _('Type') }}</th>
+            <th scope="col">{{ _('Label') }}</th>
+            <th scope="col">{{ _('Description') }}</th>
+          </tr>
+        </thead>
+        {% for f in ddict %}
+          <tr>
+            <td>{{ f.id }}</td>
+            <td>{{ f.type }}</td>
+            <td>{{ h.get_translated(f.get('info', {}), 'label') }}</td>
+            <td>{{ h.render_markdown(
+              h.get_translated(f.get('info', {}), 'notes')) }}</td>
+          </tr>
+        {% endfor %}
+      </table>
+    </div>
+  {% endif %}
+  {{ super() }}
+{% endblock %}