Final changes before end of Phase 1 Development

ckanext/cfpb_extrafields/fanstatic/ckan_overrides.css

@@ -361,6 +361,10 @@ input[type="button"].btn-block {
 }
 
 /* Footer */
+a.image, a.image:hover {
+  border: 0;
+  text-decoration: none;
+}
 
 #creeper {
   width: 50px;
@@ -425,6 +429,14 @@ a.add-new-snippet {
     width: auto;
     margin: 0;
   }
+}/* IDEA: */
+@media (min-width: 768px) {
+  /* Home Page */
+  .hero {
+    background: url("/home_backdrop.png");
+    background-repeat: no-repeat;
+    background-size: 100% 100%;
+  }
 }
 
 .expandable__padded:hover,

ckanext/cfpb_extrafields/fanstatic/datadict_stash.js

@@ -5,49 +5,54 @@ $(function() {
     $('#resource_save').click( function() {
         $( "#datadict_stash" ).trigger( "click" );
     });
+    $('#resource_save_add').click( function() {
+        $( "#datadict_stash" ).trigger( "click" );
+    });
     $('#resource_create').click( function() {
         $( "#datadict_stash" ).trigger( "click" );
     });
 });
 
 /* stashes a hidden datadict that python processes during plugin's update */
 ckan.module('datadict_stash', function ($, _) {
-    // A few jQuery helpers for exporting only
+  // A few jQuery helpers for exporting only
   jQuery.fn.pop = [].pop;
   jQuery.fn.shift = [].shift;
+
   return {
-      initialize: function () {
-          $.proxyAll(this, /_on/);
-          this.el.on('click', this._onClick);
+    initialize: function () {
+      $.proxyAll(this, /_on/);
+      this.el.on('click', this._onClick);
     },
     _onClick: function(event) {
       var $rows = $TABLE.find('tr:not(:hidden)');
       var header_keys = [];
       var header_names=[]; //head
       var record = [];
-      
+
       // Get the headers (add special header logic here)
       $($rows.shift()).find('th:not(:empty)').each(function () {
         header_keys.push($(this).text().toLowerCase());
         header_names.push($(this).text());
       });
 
-      //header_names are the first element (and remain in an ordered array)
+      // header_names are the first element (and remain in an ordered array)
       record.push(header_names);
       record.push(header_keys);
+
       // Turn all existing rows into a loopable array
       $rows.each(function () {
         var $td = $(this).find('td');
         var h = {};
-        
+
         // Use the header_keys from earlier to name our hash keys
         header_keys.forEach(function (header_key, i) {
-          h[header_key] = $td.eq(i).text();   
+          h[header_key] = $td.eq(i).text();
         });
-        
+
         record.push(h);
       });
-      
+
       // Output the result
       record = JSON.stringify(record);
       // python picks this up on submit

ckanext/cfpb_extrafields/plugin.py

@@ -1,3 +1,4 @@
+from ckan.lib.helpers import flash_error
 import ckan.plugins as p
 import ckan.plugins.toolkit as tk
 import validators as v
@@ -15,7 +16,7 @@ def create_relevant_governing_documents():
     except tk.ObjectNotFound:
         data = {'name': 'relevant_governing_documents'}
         vocab = tk.get_action('vocabulary_create')(context, data)
-        for tag in opts.relevant_governing_documents(): 
+        for tag in opts.relevant_governing_documents():
             data = {'name': tag, 'vocabulary_id': vocab['id']}
             tk.get_action('tag_create')(context, data)
 
@@ -60,7 +61,6 @@ def parse_resource_related_gist(data_related_items, resource_id):
 
 class ExampleIDatasetFormPlugin(p.SingletonPlugin, tk.DefaultDatasetForm):
 
-
     # modify ckan behavior on changes/(saves/updates/deletes) to resources
     p.implements(p.IResourceController)
     def _which_check_keys_changed(self, old, new):
@@ -75,30 +75,49 @@ def _redirect_to_edit_on_change(self, resource, field):
 
     def _delete_and_rebuild_datadict(self, resource):
         import json
+        import unicodedata
+
         if 'datadict' in resource and 'id' in resource:
-            record = resource['datadict']
-            resource.pop('datadict')
-            json_record = json.loads(record)
+            record = resource.pop('datadict')
+
+            if record:
+                # Cleanse of errant unicode characters
+                record = unicodedata.normalize('NFKD', record).encode('ascii', 'ignore')
+
+                try:
+                    json_record = json.loads(record)
+                except ValueError as err:
+                    # Invalid JSON, so don't remove old data
+                    error_message = "Error saving data dictionary: {0}.  Data was: {1}".format(err, record)
+                    flash_error(error_message)
+                    return
+
+                try:
+                    ds.delete_datastore_json(resource['id'], 'datadict')
+                # don't fail if the filter is bad! (e.g., title_colname doesn't exist)
+                except (tk.ObjectNotFound, tk.ValidationError), err:
+                    # code review: write tests for this.
+                    error_message = "Error saving data dictionary: {0}.  Data was: {1}".format(e, record)
+                    flash_error(error_message)
+                    pass
+
             try:
-                ds.delete_datastore_json(resource['id'], 'datadict')
-            # don't fail if the filter is bad! (e.g., title_colname doesn't exist)
-            except (tk.ObjectNotFound, tk.ValidationError), err:
-                # code review: write tests for this.
-                pass
-            ds.create_datastore(resource['id'], json_title='datadict', json_record=json_record)
-        return
+                ds.create_datastore(resource['id'], json_title='datadict', json_record=json_record)
+            except UnicodeEncodeError as err:
+                error_message = "Error saving data dictionary: {0}.  Data was: {1}".format(e, record)
+                flash_error(error_message)
 
     # This function is a template/starter for a more fine-grained email-on-change functionality
     # than the default notification that CKAN provides. It is not currently used.
     def _email_on_change(self, context, resource, field):
         # if specified fields have changed notify the relevant people
         if self.changed[field]:
-            # print 'trigger email on change to '+field 
-            # filter by dataset name? 
+            # print 'trigger email on change to '+field
+            # filter by dataset name?
             followers = tk.get_action('dataset_follower_list')(context,{'id': resource['package_id']})
             for f in followers:
                 # filter by group?
-                # get email addresses 
+                # get email addresses
                 print tk.get_action('user_show')(context,{'id': f['id']})['email']
                 # send a notification of change by email
 
@@ -111,24 +130,23 @@ def after_create(self, context, resource):
         # (so that users aren't forced to enter a confusing link).
         # some of that could be moved here if desired.
         return
-    
+
     def before_update(self, context, current, resource):
         # note keys that have changed (current is old, resource is new)
         self._which_check_keys_changed(current, resource)
         if current.get('resource_type', '') == 'Data Dictionary' \
            and resource.get('resource_type', '') == 'Data Dictionary':
             self._delete_and_rebuild_datadict(resource)
-            
+
     def after_update(self, context, resource):
         ''' do things on field changes '''
         # unfinished email trigger:
         # self._email_on_change(context,resource,'privacy_contains_pii')
         self._redirect_to_edit_on_change(resource, 'resource_type')
-        # reset monitored keys 
+        # reset monitored keys
         for key in self.changed:
             self.changed[key] = False
-        return
-
+
     def before_delete(self, context, resource, resources):
         return
     def after_delete(self, context, resources):
@@ -146,7 +164,7 @@ def get_helpers(self):
                 'options_legal_authority_for_collection': opts.legal_authority_for_collection,
                 'options_privacy_pia_title': opts.privacy_pia_title,
                 'options_privacy_sorn_number': opts.privacy_sorn_number,
-                'tag_relevant_governing_documents': tag_relevant_governing_documents, 
+                'tag_relevant_governing_documents': tag_relevant_governing_documents,
                 'options_relevant_governing_documents': opts.relevant_governing_documents,
                 'options_content_periodicity': opts.content_periodicity,
                 'options_update_frequency': opts.update_frequency,
@@ -165,12 +183,12 @@ def get_helpers(self):
                 'create_datastore':ds.create_datastore,
                 'get_unique_datastore_json':ds.get_unique_datastore_json,
                 'delete_datastore_json':ds.delete_datastore_json,
-                
+
                 'parse_resource_related_gist': parse_resource_related_gist,
                 'github_api_url': github_api_url,
             }
 
-    
+
     # the main extra fields update/show functionality
     p.implements(p.IDatasetForm)
     def _modify_package_schema(self, schema):
@@ -225,7 +243,7 @@ def _modify_package_schema(self, schema):
                                 tk.get_converter('convert_to_extras')],
             'content_temporal_range_end' : [v.end_after_start_validator, v.reasonable_date_validator,
                                             tk.get_validator('ignore_missing'),
-                                            tk.get_converter('convert_to_extras')], 
+                                            tk.get_converter('convert_to_extras')],
             'content_temporal_range_start' : [v.end_after_start_validator, v.reasonable_date_validator,
                                               tk.get_validator('ignore_missing'),
                                               tk.get_converter('convert_to_extras')],
@@ -278,8 +296,8 @@ def _modify_package_schema(self, schema):
                 'resource_type' : [tk.get_validator('ignore_missing'),],
                 'storage_location' : [tk.get_validator('ignore_missing'),],
                 'storage_location_path' : [tk.get_validator('ignore_missing'),],
-                'database_server' : [ tk.get_validator('ignore_missing'),],  
-                'database_name' : [ tk.get_validator('ignore_missing'),],  
+                'database_server' : [ tk.get_validator('ignore_missing'),],
+                'database_name' : [ tk.get_validator('ignore_missing'),],
                 'database_schema' : [ tk.get_validator('ignore_missing'),],
                 'db_role_level_1' : [ tk.get_validator('ignore_missing'),],
                 'db_role_level_2' : [ tk.get_validator('ignore_missing'),],
@@ -292,17 +310,17 @@ def _modify_package_schema(self, schema):
                 'db_role_level_9' : [ tk.get_validator('ignore_missing'),],
         })
         return schema
-    
+
     def create_package_schema(self):
         schema = super(ExampleIDatasetFormPlugin, self).create_package_schema()
         schema = self._modify_package_schema(schema)
         return schema
-    
+
     def update_package_schema(self):
         schema = super(ExampleIDatasetFormPlugin, self).update_package_schema()
         schema = self._modify_package_schema(schema)
         return schema
-    
+
     def show_package_schema(self):
         schema = super(ExampleIDatasetFormPlugin, self).show_package_schema()
         schema.update({
@@ -392,9 +410,9 @@ def show_package_schema(self):
                 'intake_date' : [tk.get_validator('ignore_missing'),],
                 'resource_type' : [ tk.get_validator('ignore_missing'),],
                 'storage_location' : [ tk.get_validator('ignore_missing'),],
-                'storage_location_path' : [ tk.get_validator('ignore_missing'),],  
-                'database_server' : [ tk.get_validator('ignore_missing'),],  
-                'database_name' : [ tk.get_validator('ignore_missing'),],  
+                'storage_location_path' : [ tk.get_validator('ignore_missing'),],
+                'database_server' : [ tk.get_validator('ignore_missing'),],
+                'database_name' : [ tk.get_validator('ignore_missing'),],
                 'database_schema' : [ tk.get_validator('ignore_missing'),],
                 'db_role_level_1' : [ tk.get_validator('ignore_missing'),],
                 'db_role_level_2' : [ tk.get_validator('ignore_missing'),],
@@ -415,18 +433,18 @@ def show_package_schema(self):
                 tk.get_validator('ignore_missing')]
             })
         return schema
-    
+
     def is_fallback(self):
         # Return True to register this plugin as the default handler for
         # package types not handled by any other IDatasetForm plugin.
         return True
-    
+
     def package_types(self):
         # This plugin doesn't handle any special package types, it just
         # registers itself as the default (above).
         return []
 
-    
+
     p.implements(p.IConfigurer)
     def update_config(self, config):
         # Add this plugin's templates dir to CKAN's extra_template_paths, so
@@ -460,7 +478,7 @@ def _change_facets(self, facets_dict):
         facets_dict['res_type'] = p.toolkit._('Resource Types')
         facets_dict['res_format'] = p.toolkit._('Formats')
         return facets_dict
-    
+
     # now return the same altered search facets for the dataset, group and organization page
     def dataset_facets(self, facets_dict, package_type):
         return self._change_facets(facets_dict)

ckanext/cfpb_extrafields/templates/footer.html

@@ -2,7 +2,7 @@
 
 {% block footer_links_ckan %}
 {% set api_url = 'http://docs.ckan.org/en/{0}/api/'.format(g.ckan_doc_version) %}
-<li><a href="{{ api_url }}">{{ _('CKAN API') }}</a></li> 
-<li><a href="http://team.cfpb.local/wiki/index.php/Data_Team"><img src="/doozer.png" width="50" id="creeper"> Data Team</a></li>
+<li><a href="{{ api_url }}">{{ _('CKAN API') }}</a></li>
+<li><a href="http://team.cfpb.local/wiki/index.php/Data_Team">Questions? Ask the Data Team</a></li>
+<li><a href="http://team.cfpb.local/wiki/index.php/Data_Team"><img src="/doozer.png" width="50" id="creeper"></a></li>
 {% endblock %}
-

ckanext/cfpb_extrafields/templates/home/snippets/search.html

@@ -0,0 +1,21 @@
+{% set tags = h.get_facet_items_dict('tags', limit=3) %}
+{% set placeholder = _('E.g. mortgage') %}
+
+<div class="module module-search module-narrow module-shallow box">
+  <form class="module-content search-form" method="get" action="{% url_for controller='package', action='search' %}">
+    <h3 class="heading">{{ _("Search data") }}</h3>
+    <div class="search-input control-group search-giant">
+      <input type="text" class="search" name="q" value="" autocomplete="off" placeholder="{% block search_placeholder %}{{ placeholder }}{% endblock %}" />
+      <button type="submit">
+        <i class="icon-search"></i>
+        <span>{{ _('Search') }}</span>
+      </button>
+    </div>
+  </form>
+  <div class="tags">
+    <h3>{{ _('Popular tags') }}</h3>
+    {% for tag in tags %}
+      <a class="tag" href="{% url_for controller='package', action='search', tags=tag.name %}">{{ h.truncate(tag.display_name, 22) }}</a>
+    {% endfor %}
+  </div>
+</div>