CDO phase1 -- update fields

ckanext/cfpb_extrafields/exportutils.py

@@ -16,7 +16,6 @@
     ("title", "Dataset title"),
     ("notes", "Description"),
     ("private", "Visibility"),
-    # ("i", "Subject Matter"),
     ("organization.name", "Organization"),
     ("also_known_as", "Also known as"),
     ("data_source_names", "Data sources"),
@@ -36,7 +35,6 @@
     ("access_notes", "How to get access"),
     ("usage_restrictions", "Usage restrictions"),
     ("dataset_notes", "Dataset notes"),
-    # ("i", "Dataset Last Modified Date"),
     ("obfuscated_title", "Obfuscated title"),
     ("transfer_details", "Transfer details"),
     ("transfer_initial_size", "Transfer initial size (MB)"),
@@ -62,6 +60,16 @@
     ("legal_notes","Legal: Notes"),
     ("pra_notes","PRA: Notes"),
     ("privacy_notes","Privacy: Notes"),
+    # New
+    ("website_name", "Reference Website Name"),
+    ("cleansing_rules_used", "cleansing rules used"),
+    ("dataset_last_modified_date", "Dataset Last Modified Date"),
+    ("foia_exemptions", "FOIA: Exemptions"),
+    ("foia_exemptions_notes", "FOIA Exemption Notes"),
+    ("opendata_indicator", "Open Data: open data indicator"),
+    ("opendata_priority_indicator", "Open Data: priority indicator"),
+    ("opendata_public_location", "Open Data: public location"),
+
 ]
 
 try:

ckanext/cfpb_extrafields/options.py

@@ -92,6 +92,23 @@ def relevant_governing_documents():
 def content_spatial():
     return [""]
 
+# WARNING - Updates will not update stored data
+def foia_exemptions():
+    return [
+            "b3 - Specifically prohibited from disclosure by another federal statute",
+            "b4 - Trade secrets, commercial, financial, privileged or confidential",
+            "b5 - Privileged inter-agency or intra-agency communications",
+            "b6 - Personnel, medical files, or similar information. Includes all PII",
+            "b7A - Law enforcement data protected for law enforcement proceedings",
+            "b7B - Law enforcement data protected for fair trial or adjudication",
+            "b7C - Law enforcement data protected for personal privacy",
+            "b7D - Law enforcement data protected to safeguard a confidential source",
+            "b7E - Law enforcement data protected to avoid circumvention of the law",
+            "b7F - Law enforcement data protected to safeguard life or physical safety",
+            "b8 - Financial institutions supervision data",
+            "No Exemption",
+            ]
+
 def frequency_standards():
 #http://dublincore.org/groups/collections/frequency/
     a=["Triennial", "Biennial","Annual",

ckanext/cfpb_extrafields/plugin.py

@@ -202,6 +202,7 @@ def get_helpers(self):
                 'options_approximate_total_size': opts.approximate_total_size,
                 'options_resource_type': opts.resource_type,
                 'options_source_categories': opts.source_categories,
+                'options_foia_exemptions': opts.foia_exemptions,
                 'create_datastore': ds.create_datastore,
                 'get_unique_datastore_json': ds.get_unique_datastore_json,
                 'delete_datastore_json': ds.delete_datastore_json,
@@ -322,6 +323,17 @@ def _modify_package_schema(self, schema):
                                                 tk.get_converter('convert_to_extras'),],
             'privacy_notes' : [tk.get_validator('ignore_missing'),
                                                 tk.get_converter('convert_to_extras'),],
+            'opendata_indicator' : [tk.get_validator('ignore_missing'),
+                                                tk.get_converter('convert_to_extras'),],
+            'opendata_priority_indicator' : [tk.get_validator('ignore_missing'),
+                                                tk.get_converter('convert_to_extras'),],
+            'opendata_public_location' : [tk.get_validator('ignore_missing'),
+                                                tk.get_converter('convert_to_extras'),],
+            'foia_exemptions': [tk.get_validator('ignore_missing'),
+                        v.input_value_validator,
+                        tk.get_converter('convert_to_extras')],
+            'foia_exemptions_notes' : [tk.get_validator('ignore_missing'),
+                                                tk.get_converter('convert_to_extras'),],
         })
         # now modify tag fields and convert_to_tags
         schema.update({
@@ -458,6 +470,12 @@ def show_package_schema(self):
                             tk.get_validator('ignore_missing'),],
             'privacy_notes' : [tk.get_converter('convert_from_extras'),
                                tk.get_validator('ignore_missing'),],
+            'opendata_indicator' : [ tk.get_converter('convert_from_extras'),tk.get_validator('ignore_missing'),],
+            'opendata_priority_indicator' : [ tk.get_converter('convert_from_extras'),tk.get_validator('ignore_missing'),],
+            'opendata_public_location' : [ tk.get_converter('convert_from_extras'),tk.get_validator('ignore_missing'),],
+            'foia_exemptions': [tk.get_converter('convert_from_extras'),
+                                    tk.get_validator('ignore_missing')],
+            'foia_exemptions_notes' : [ tk.get_converter('convert_from_extras'),tk.get_validator('ignore_missing'),],
         })
         schema['resources'].update({
                 'approximate_total_size' : [ tk.get_validator('ignore_missing'),],

ckanext/cfpb_extrafields/templates/macros/form_custom.html

@@ -25,7 +25,7 @@
 {% endmacro %}
 
 {# select multiple list with specified other option #}
-{% macro select_multi_with_other(name, id='', label='', current_values=[], size="400", options_defaults=[], extra_html="", error='') %}
+{% macro select_multi_with_other(name, id='', label='', current_values=[], size="400", options_defaults=[], extra_html="", error='',other=True) %}
 <div class="control-group{{ " error" if error }}{{ " " ~ classes | join(' ') }}">
     <label class="control-label" for="{{ name }}">{{label}}</label>
     <div class="controls{{ " " ~ control_classes | join(' ') }}">
@@ -43,7 +43,9 @@
         {% endif %}
     {% endfor %}
     {# provide an option for javascript to use to create new fields #}
+    {% if other %}
     <option value="__Other">Other (please specify)</option>
+    {% endif %}
     </select>
     {% if error and error is iterable %}<span class="error-block">{{ error|join(', ') }}</span>
     {% elif error %} {{error}} {% endif %}

ckanext/cfpb_extrafields/templates/package/snippets/additional_info.html

@@ -82,6 +82,12 @@
             <td class="dataset-details"><a href="{{ pkg_dict.website_url }}">{{ pkg_dict.website_name}}</a> </td>
           </tr>
           {% endif %}
+	  {% if pkg_dict.dataset_last_modified_date %}
+	  <tr>
+		  <th scope="row" class="dataset-label">{{ _("Dataset Last Modified Date") }}</th>
+		  <td class="dataset-details">{{ pkg_dict.dataset_last_modified_date }}</td>
+	  </tr>
+	  {% endif %}
         </tbody>
       </table>
     </div>
@@ -177,12 +183,6 @@
             <td class="dataset-details">{{ h.render_markdown(pkg_dict.dataset_notes) }}</td>
           </tr>
         {% endif %}
-        {% if pkg_dict.dataset_last_modified_date %}
-          <tr>
-            <th scope="row" class="dataset-label">{{ _("Dataset Last Modified Date") }}</th>
-            <td class="dataset-details">{{ pkg_dict.dataset_last_modified_date }}</td>
-          </tr>
-        {% endif %}
         {% if pkg_dict.obfuscated_title %}
           <tr>
             <th scope="row" class="dataset-label">{{ _("Obfuscated Title") }}</th>
@@ -356,6 +356,18 @@
             <td class="dataset-details">{{ pkg_dict.procurement_document_id }}</td>
           </tr>
         {% endif %}
+        {% if pkg_dict.foia_exemptions %}
+          <tr>
+            <th scope="row" class="dataset-label">{{ _("FOIA Exemptions") }}</th>
+            <td class="dataset-details">{% for i in h.clean_select_multi(pkg_dict.foia_exemptions) %} {{ i }}{% if not loop.last %},{% endif %} {% endfor %}</td>
+          </tr>
+        {% endif %}
+        {% if pkg_dict.foia_exemptions_notes %}
+          <tr>
+            <th scope="row" class="dataset-label">{{ _("FOIA Exemption Notes") }}</th>
+            <td class="dataset-details">{{ h.render_markdown(pkg_dict.foia_exemptions_notes) }}</td>
+          </tr>
+        {% endif %}
         {% if pkg_dict.cleansing_rules_used and false %}
         <tr>
           <th scope="row" class="dataset-label">{{ _("Cleansing Rules Used") }}</th>
@@ -368,6 +380,49 @@
     </div>
   </div>
 
+  <div class="expandable expandable__padded">
+    <button class="expandable_header expandable_target" title="Expand content">
+        <span class="expandable_header-left expandable_label">
+            V. Open Data
+        </span>
+        <span class="expandable_header-right expandable_link">
+            <span class="expandable_cue-open">
+                Show
+                <span class="cf-icon cf-icon-plus-round"></span>
+            </span>
+            <span class="expandable_cue-close">
+                Hide
+                <span class="cf-icon cf-icon-minus-round"></span>
+            </span>
+        </span>
+    </button>
+    <div class="expandable_content">
+      <table class="table table-striped table-bordered table-condensed">
+        <tbody>
+        {% if pkg_dict.opendata_indicator %}
+         <tr>
+            <th scope="row" class="dataset-label">{{_("Open Data Indicator") }}</th>
+            <td class="dataset-details">{{ pkg_dict.opendata_indicator }}</td>
+          </tr>
+        {% endif %}
+        {% if pkg_dict.opendata_priority_indicator %}
+         <tr>
+            <th scope="row" class="dataset-label">{{_("Priority Indicator") }}</th>
+            <td class="dataset-details">{{ pkg_dict.opendata_priority_indicator }}</td>
+          </tr>
+        {% endif %}
+        {% if pkg_dict.opendata_public_location %}
+         <tr>
+            <th scope="row" class="dataset-label">{{_("Public Location") }}</th>
+            <td class="dataset-details">{{ pkg_dict.opendata_public_location }}</td>
+          </tr>
+        {% endif %}
+
+        </tbody>
+      </table>
+    </div>
+  </div>
+
 </section>
 {% endblock %}
 {% endblock %}

ckanext/cfpb_extrafields/templates/package/snippets/package_metadata_fields.html

@@ -52,6 +52,8 @@ <h3>I. Content</h3>
     <input id="field-website_name" type="text" placeholder="name" value="{{data.website_name}}" name="website_name" />
 {% endcall %}
 
+{{ form.input('dataset_last_modified_date', type='date', label=_("Dataset Last Modified Date"), placeholder="eg 2012-12-21", value=data.dataset_last_modified_date, error=errors.dataset_last_modified_date) }}
+
 <hr>
 
 <h3>II. Access</h3>
@@ -87,8 +89,6 @@ <h3>III. Administrative</h3>
 
 {{ form.markdown('dataset_notes', id='field-dataset_notes', label=_('Dataset Notes'), placeholder=_(''), value=data.dataset_notes, error=errors.dataset_notes) }}
 
-{{ form.input('dataset_last_modified_date', type='date', label=_("Dataset Last Modified Date"), placeholder="eg 2012-12-21", value=data.dataset_last_modified_date, error=errors.dataset_last_modified_date) }}
-
 {{ form_custom.radio('obfuscated_title', label=_('Obfuscated Title'), current_value=data.obfuscated_title) }}
 
 {{ form.markdown('transfer_details', id='field-transfer_details', label=_('Transfer Details'), placeholder=_('enter text'), value=data.transfer_details, error=errors.transfer_details) }}
@@ -218,12 +218,40 @@ <h3>IV. Compliance</h3>
 
 {{ form.markdown('records_retention_schedule', id='field-records_retention_schedule', label=_('records retention schedule'), placeholder=_(''), value=data.records_retention_schedule, error=errors.records_retention_schedule) }}
 
+{{ form_custom.select_multi_with_other('foia_exemptions',
+label=_('FOIA: Exemptions'),
+current_values=h.clean_select_multi(data.get('foia_exemptions', '')),
+options_defaults=h.options_foia_exemptions(),
+error=errors.foia_exemptions,
+size="550",
+other=False,
+) }}
+
+<div>
+{{ form.markdown('foia_exemptions_notes', id='field-foia_exemptions_notes', label=_('FOIA Exemption Notes'), placeholder=_(''), value=data.foia_exemptions_notes, error=errors.foia_exemptions_notes) }}
+</div>
+
 {{ form.input('procurement_document_id', label=_("procurement document ID"), placeholder="CFPB-12-Z-00015-0011", value=data.procurement_document_id, error=errors.procurement_document_id) }}
 
 {% if false %}
 {{ form.markdown('cleansing_rules_used', id='field-cleansing_rules_used', label=_('cleansing rules used'), placeholder=_(''), value=data.cleansing_rules_used, error=errors.cleansing_rules_used) }}
 {% endif %}
 
+<hr>
+<h3>V. Open Data</h3>
+
+{{ form_custom.radio('opendata_indicator',
+label=_('Open Data: open data indicator'),
+current_value=data.opendata_indicator)}}
+
+{{ form_custom.radio('opendata_priority_indicator',
+label=_('Open Data: priority indicator'),
+current_value=data.opendata_priority_indicator)}}
+
+{{ form.input('opendata_public_location',
+label=_('Open Data: public location'),
+value=data.opendata_public_location,
+error=errors.opendata_public_location)}}
 
 <script>
   // NOTES: less than IE9 support
@@ -270,6 +298,11 @@ <h3>IV. Compliance</h3>
       "transfer_initial_size": "Approximate size of the dataset when brought into the Bureau (in megabytes)",
       "transfer_method": "Technique, format, or medium for bringing the data into the bureau",
       "update_frequency": "How often the Bureau receives updates to the data",
+      "foia_exemptions": "Would this data be available through a FOIA request? If yes, coose \"No Exemption\". If no, indicate the FOIA exemptions that would prevent publication of this data asset",
+      "foia_exemptions_notes": "If a FOIA Exemption is supported, please list any supporting information available such as law that prvents publication, privacy agreement, Memo of Understanding with a company or agency, subset of data that can be published if applicable",
+      "opendata_indicator": "Can the Bureau publish this data asset to Data.gov and ConsumerFinance.gov?",
+      "opendata_priority_indicator": "Has this data asset been added to the Bureau's priority data asset due to the value it can provide to external customers, its prior publication on ConsumerFinance.gov, or data assets recommended for publication by customer facing Bureau organizations?",
+      "opendata_public_location": "What is the URL where the general public can access the data assets, its data dictionary, JSON, XML, and or API?",
       "usage_restrictions": "Rules and restrictions for using the specific dataset, above and beyond normal Bureau policies and applicable law. All usage and disclosure of information is subject to applicable law, including the Bureau’s confidentiality regulations, the Privacy Act, and the Trade Secrets Act.  Information should not be disclosed outside of the Bureau unless it is aggregated and does not directly or indirectly identify an individual or an entity.  Additional usage or disclosure restrictions may be present in contracts, MOUs, NDAs, or similar agreements.",
       "website_url": "Link to a relevant website",
       "wiki_link": "Link to a CFPB wiki page for this dataset"

ckanext/cfpb_extrafields/tests.py

@@ -15,8 +15,10 @@
 class TestValidators(unittest.TestCase):
     maxDiff=None
     @parameterized.expand([
-        (u'"asdf","asdf,asdf"',[u'asdf', u'asdf,asdf']),
+        #(u'"asdf","asdf,asdf"',[u'asdf', u'asdf,asdf']),
+        (u'"asdf","asdf,asdf"',[u'"asdf","asdf,asdf"']),
         (u'asdf',[u'asdf']),
+        (u'asdf, asdf',[u'asdf, asdf']),
         (u'a',[u'a']),
         (u'{"blah blah","blah asdf",asdf}',[u'blah blah', u'blah asdf', u'asdf']),
         (u'{asdf,asdf}',[u'asdf', u'asdf']),
@@ -26,6 +28,21 @@ class TestValidators(unittest.TestCase):
         (u'{a,"f d",d}',[u'a', u'f d', u'd']),
         (u'',[]),
         (["foo"],["foo"]),
+        (u'''{123Consumers,123-Consumers,"123,Consumers",123.Consumers,123?Consumers,123<>Consumers,"123{}Consumers",123[]Consumers,123()Consumers,"123\\Consumers",123+Consumers,123'Consumers}''',
+        [
+            u'123Consumers',
+            u'123-Consumers',
+            u"123,Consumers",
+            u'123.Consumers',
+            u'123?Consumers',
+            u'123<>Consumers',
+            u"123{}Consumers",
+            u'123[]Consumers',
+            u'123()Consumers',
+            u"123\\Consumers",
+            u'123+Consumers',
+            u"123'Consumers"]
+            ),
     ])
     def test_clean_select_multi(self, ms, expected):
         assert_equal(v.clean_select_multi(ms), expected)

ckanext/cfpb_extrafields/validators.py

@@ -1,5 +1,6 @@
 import re
 import datetime
+import json
 
 # Abstracted to simplify unit testing via mock
 def Invalid(message):
@@ -81,17 +82,18 @@ def end_after_start_validator(key, flattened_data, errors, context):
             Invalid("content start date occurs after end date")
     return
 
-
 # select multis are contained in unicode strings that look like:
 # u'{"blah blah","blah asdf",asdf}' ; u'{asdf,asdf}' ; u'asdf' (see also tests.py)
-def clean_select_multi(s):
+def clean_select_multi(raw_s):
     ''' parses the results of an html form select-multi '''
     # This solution allows commas, but is unpythonic
-    if not s:
+    if not raw_s:
         return []
-    if not isinstance(s, basestring):
-        return s
-    s = s.lstrip('{').rstrip('}')
+    if not isinstance(raw_s, basestring):
+        return raw_s
+    s = raw_s.lstrip('{').rstrip('}')
+    if s == raw_s:
+        return [ s ]
     clean = []
     left = 0
     right = 1