Merge pull request #66 from dvb-cfpb/fix-dig-import-2021

Fix dig import 2021

ckanext/cfpb_extrafields/controllers/digimport.py

@@ -31,12 +31,16 @@ def upload(self):
         dig = request.POST["file"].file
         group = request.POST["group"]
         rec, errors = make_rec(dig)
-        if errors:
+
+        # The following takes DIG processing "errors" fields impeding intake
+        # per CDO we are allowing all DIG values through "as is" and letting
+        # CKAN do the error processing/reporting.
+        if False: #errors:
             redirect_to("import_page", errors=json.dumps(errors), group=group)
         else:
             rec["owner_org"] = group
-            rec["name"] = make_name(request.POST.get("name") or rec["title"])
-            rec["notes"] = rec["notes"] or "Record automatcially created from DIG file"
+            rec["name"] = make_name(request.POST.get("name") or rec.get("title",""))
+            rec["notes"] = rec.get("notes","") or "Record automatcially created from DIG file"
             try:
                 upload_rec(rec)
             except ValidationError as err:

ckanext/cfpb_extrafields/digutils.py

@@ -2,12 +2,6 @@
 import re
 
 from openpyxl import load_workbook
-try:
-    from ckan.plugins.toolkit import Invalid
-except ImportError: # pragma: no cover
-    # If the custom exception can't be imported, use a more generic exception
-    # This happens when ckan is not installed locally, like when running unit tests on travis.
-    Invalid = Exception
 
 from ckanext.cfpb_extrafields import validators as v
 
@@ -194,9 +188,13 @@ def make_rec_from_sheet(ws, fields):
     for field in fields:
         try:
             result[field] = get_field(ws, field, fields)
-        except Invalid as  err:
+        except NotFound as  err:
+            errors.append(field + ": Unable to extract field from workbook - Check for duplicate or undefined cell ranges")
+        except (Exception, StopIteration) as  err:
+            # Invalid or Not Found respectively
             errors.append(field + ": " + getattr(err, "error", getattr(err, "message", "UKNOWN_ERROR")))
     return result, errors
+
 def make_rec(excel_file):
     wb = load_workbook(excel_file, read_only=True)
     version = get_schema_version(wb)