Added fixes for data dictionary bugs

ckanext/cfpb_extrafields/plugin.py

@@ -75,15 +75,20 @@ def _redirect_to_edit_on_change(self, resource, field):
 
     def _delete_and_rebuild_datadict(self, resource):
         import json
+        import unicodedata
+
         if 'datadict' in resource and 'id' in resource:
             record = resource.pop('datadict')
 
             if record:
+                # Cleanse of errant unicode characters
+                record = unicodedata.normalize('NFKD', record).encode('ascii', 'ignore')
+
                 try:
                     json_record = json.loads(record)
-                except ValueError as e:
+                except ValueError as err:
                     # Invalid JSON, so don't remove old data
-                    error_message = "Error saving data dictionary: {0}.  Data was: {1}".format(e, record)
+                    error_message = "Error saving data dictionary: {0}.  Data was: {1}".format(err, record)
                     flash_error(error_message)
                     return
 
@@ -96,8 +101,11 @@ def _delete_and_rebuild_datadict(self, resource):
                     flash_error(error_message)
                     pass
 
-            ds.create_datastore(resource['id'], json_title='datadict', json_record=json_record)
-        return
+            try:
+                ds.create_datastore(resource['id'], json_title='datadict', json_record=json_record)
+            except UnicodeEncodeError as err:
+                error_message = "Error saving data dictionary: {0}.  Data was: {1}".format(e, record)
+                flash_error(error_message)
 
     # This function is a template/starter for a more fine-grained email-on-change functionality
     # than the default notification that CKAN provides. It is not currently used.

ckanext/cfpb_extrafields/templates/package/snippets/resource_type-specific/cfpb_extrafields_datadictionary.html

@@ -14,23 +14,23 @@
   {% else %}
     {{ form.hidden('url', value=" optional URL or click (X) to upload a file")}}
   {% endif %}
-  
+
   <div class='core-field'>
-  {{ form.markdown('description', id='field-description', label=_('description'), 
+  {{ form.markdown('description', id='field-description', label=_('description'),
   placeholder=_('URL to a related resource or other notes about the data dictionary'), value=data.description,
   error=errors.description) }}
   </div>
-  
+
   <!-- format persists unless reset -->
   <div hidden class='core-field'>
   <input id="format" value="data dictionary" name="format"/>
   </div>
   <!-- end cfpb_extrafields_datadictionary.html snippet -->
 {% endif %}
-  
+
 {% if location=="form" or location=="description" %}
   <!-- cfpb_extrafields_datadictionary.html snippet -->
-  
+
 {% resource 'cfpb_extrafields/datadict_setup.css' %}
 {% resource 'cfpb_extrafields/datadict_setup.js' %}
 {% resource 'cfpb_extrafields/datadict_stash.js' %}
@@ -48,14 +48,14 @@
 {% set nkeys = range(dd[1]|length) %}
 {% set nrows = dd|length-2 %}
 
-<div id="datadict-table" class="table-editable">    
+<div id="datadict-table" class="table-editable">
   {% if editable == "true" %}
   <span class="table-add"><button type='button' class="btn btn-primary">+</button></span>
   {% endif %}
   <table class="table">
     <tr>
       {% for item in header_names %}
-        <th><div contenteditable={{editable}}>{{ item }}</div></th>
+        <th {% if item == "Definition" %}style="width: 12em"{% endif %}><div contenteditable={{editable}}>{{ item }}</div></th>
       {% endfor %}
       {% if editable == "true" %}
       <th></th>
@@ -65,8 +65,8 @@
   {% for irow in range(2,nrows+2) %}
     <tr>
       {% for i in nkeys %}
-        {% set key = header_keys[i]%}
-          <td><div contenteditable={{editable}}>{{ dd[irow][key] }}</div></td>
+        {% set key = header_keys[i] %}
+          <td {% if key == "definition" %}style="width: 12em"{% endif %}><div contenteditable={{editable}}>{{ dd[irow][key] }}</div></td>
         {% endfor %}
       {% if editable == "true" %}
       <td>
@@ -97,13 +97,13 @@
 
 <input type="hidden" id="field-datadict" name="datadict">
 <button class="btn btn-primary" type="button" style="display: none;"
-        id="datadict_stash" 
+        id="datadict_stash"
         data-module="datadict_stash"
         data-module-dataset_id="{{ data.package_id }}"
         data-module-resource_id="{{ data.id }}"
         data-module-resource_name="{{ data.name }}">Stash hidden JSON of dictionary</button>
 <!-- end cfpb_extrafields_datadictionary.html snippet -->
 {% endif %}
 {% if location=="description" %}<div align="right">
-  <i class="icon-info-sign"><a href='/api/3/action/datastore_search?resource_id={{data.id}}&filters={"record_title":"datadict"}'>{{ _(' JSON format') }}</a></i></div>
+  <i class="icon-info-sign" title="After exporting to JSON, you can convert the data dictionary to other formats (like csv) to allow for easier use."><a href='/api/3/action/datastore_search?resource_id={{data.id}}&filters={"record_title":"datadict"}'>{{ _(' Export to JSON') }}</a></i></div>
 {% endif %}