Merge branch '2671-fix-partial-updates'

ckan/lib/dictization/model_save.py

@@ -59,7 +59,7 @@ def resource_dict_save(res_dict, context):
 
 def package_resource_list_save(res_dicts, package, context):
     allow_partial_update = context.get("allow_partial_update", False)
-    if not res_dicts and allow_partial_update:
+    if res_dicts is None and allow_partial_update:
         return
 
     pending = context.get('pending')
@@ -68,7 +68,7 @@ def package_resource_list_save(res_dicts, package, context):
     old_list = package.resource_groups_all[0].resources_all[:]
 
     obj_list = []
-    for res_dict in res_dicts:
+    for res_dict in res_dicts or []:
         obj = resource_dict_save(res_dict, context)
         obj_list.append(obj)
 
@@ -87,7 +87,7 @@ def package_resource_list_save(res_dicts, package, context):
 
 def package_extras_save(extra_dicts, obj, context):
     allow_partial_update = context.get("allow_partial_update", False)
-    if not extra_dicts and allow_partial_update:
+    if extra_dicts is None and allow_partial_update:
         return
 
     model = context["model"]
@@ -98,7 +98,7 @@ def package_extras_save(extra_dicts, obj, context):
     old_extras = dict((extra.key, extra) for extra in extras_list)
 
     new_extras = {}
-    for extra_dict in extra_dicts:
+    for extra_dict in extra_dicts or []:
         if extra_dict.get("deleted"):
             continue
 
@@ -151,7 +151,7 @@ def group_extras_save(extras_dicts, context):
 
 def package_tag_list_save(tag_dicts, package, context):
     allow_partial_update = context.get("allow_partial_update", False)
-    if not tag_dicts and allow_partial_update:
+    if tag_dicts is None and allow_partial_update:
         return
 
     model = context["model"]
@@ -169,7 +169,7 @@ def package_tag_list_save(tag_dicts, package, context):
 
     tag_name_vocab = set()
     tags = set()
-    for tag_dict in tag_dicts:
+    for tag_dict in tag_dicts or []:
         if (tag_dict.get('name'), tag_dict.get('vocabulary_id')) not in tag_name_vocab:
             tag_obj = d.table_dict_save(tag_dict, model.Tag, context)
             tags.add(tag_obj)
@@ -202,7 +202,7 @@ def package_tag_list_save(tag_dicts, package, context):
 def package_membership_list_save(group_dicts, package, context):
 
     allow_partial_update = context.get("allow_partial_update", False)
-    if not group_dicts and allow_partial_update:
+    if group_dicts is None and allow_partial_update:
         return
 
     capacity = 'public'
@@ -216,7 +216,7 @@ def package_membership_list_save(group_dicts, package, context):
                          for member in
                          members)
     groups = set()
-    for group_dict in group_dicts:
+    for group_dict in group_dicts or []:
         id = group_dict.get("id")
         name = group_dict.get("name")
         capacity = group_dict.get("capacity", "public")
@@ -255,8 +255,9 @@ def package_membership_list_save(group_dicts, package, context):
 def relationship_list_save(relationship_dicts, package, attr, context):
 
     allow_partial_update = context.get("allow_partial_update", False)
-    if not relationship_dicts and allow_partial_update:
+    if relationship_dicts is None and allow_partial_update:
         return
+
     model = context["model"]
     session = context["session"]
     pending = context.get('pending')
@@ -265,7 +266,7 @@ def relationship_list_save(relationship_dicts, package, attr, context):
     old_list = relationship_list[:]
 
     relationships = []
-    for relationship_dict in relationship_dicts:
+    for relationship_dict in relationship_dicts or []:
         obj = d.table_dict_save(relationship_dict,
                               model.PackageRelationship, context)
         relationships.append(obj)
@@ -297,20 +298,20 @@ def package_dict_save(pkg_dict, context):
     if not pkg.id:
         pkg.id = str(uuid.uuid4())
 
-    package_resource_list_save(pkg_dict.get("resources", []), pkg, context)
-    package_tag_list_save(pkg_dict.get("tags", []), pkg, context)
-    package_membership_list_save(pkg_dict.get("groups", []), pkg, context)
+    package_resource_list_save(pkg_dict.get("resources"), pkg, context)
+    package_tag_list_save(pkg_dict.get("tags"), pkg, context)
+    package_membership_list_save(pkg_dict.get("groups"), pkg, context)
 
     # relationships are not considered 'part' of the package, so only
     # process this if the key is provided
     if 'relationships_as_subject' in pkg_dict:
-        subjects = pkg_dict.get('relationships_as_subject', [])
+        subjects = pkg_dict.get('relationships_as_subject')
         relationship_list_save(subjects, pkg, 'relationships_as_subject', context)
     if 'relationships_as_object' in pkg_dict:
-        objects = pkg_dict.get('relationships_as_object', [])
+        objects = pkg_dict.get('relationships_as_object')
         relationship_list_save(objects, pkg, 'relationships_as_object', context)
 
-    extras = package_extras_save(pkg_dict.get("extras", []), pkg, context)
+    extras = package_extras_save(pkg_dict.get("extras"), pkg, context)
 
     return pkg
 

ckan/lib/navl/dictization_functions.py

@@ -2,6 +2,7 @@
 import formencode as fe
 import inspect
 from pylons.i18n import _
+from pylons import config
 
 class Missing(object):
     def __unicode__(self):
@@ -217,8 +218,22 @@ def validate(data, schema, context=None):
     context = context or {}
 
     assert isinstance(data, dict)
+
+    # store any empty lists in the data as they will get stripped out by
+    # the _validate function. We do this so we can differentiate between
+    # empty fields and missing fields when doing partial updates.
+    empty_lists = [key for key, value in data.items() if value == []]
+
     flattened = flatten_dict(data)
     converted_data, errors = _validate(flattened, schema, context)
+
+    # check config for partial update fix option
+    if config.get('ckan.fix_partial_updates', False):
+        # repopulate the empty lists
+        for key in empty_lists:
+            if key not in converted_data:
+                converted_data[(key,)] = []
+
     converted_data = unflatten(converted_data)
 
     errors_unflattened = unflatten(errors)

ckan/tests/lib/test_dictization_schema.py

@@ -64,14 +64,14 @@ def test_1_package_schema(self):
 
         self.remove_changable_columns(result)
 
-        pprint(result)
-
         result['name'] = 'anna2'
+        # we need to remove these as they have been added
+        del result['relationships_as_object']
+        del result['relationships_as_subject']
 
         converted_data, errors = validate(result, default_package_schema(), self.context)
 
 
-        pprint(errors)
         assert converted_data == {'extras': [{'key': u'genre', 'value': u'"romantic novel"'},
                                             {'key': u'original media', 'value': u'"book"'}],
                                    'groups': [{u'name': u'david',
@@ -141,6 +141,12 @@ def test_2_group_schema(self):
 
         data = group_dictize(group, self.context)
 
+        # we don't want these here
+        del data['groups']
+        del data['users']
+        del data['tags']
+        del data['extras']
+
         converted_data, errors = validate(data, default_group_schema(), self.context)
         group_pack = sorted(group.active_packages().all(), key=lambda x:x.id)