Merge pull request #485 from OpenDataServices/476-bad-duplicate-ids

[#476] Don't 500 error on duplicate IDs with unexpected types

cove/lib/common.py

@@ -29,7 +29,7 @@ def uniqueIds(validator, uI, instance, schema):
             except AttributeError:
                 # if item is not a dict
                 item_id = None
-            if item_id and not isinstance(item_id, list):
+            if item_id and not isinstance(item_id, list) and not isinstance(item_id, dict):
                 if item_id in all_ids:
                     non_unique_ids.add(item_id)
                 all_ids.add(item_id)
@@ -41,7 +41,7 @@ def uniqueIds(validator, uI, instance, schema):
                 return
 
         if non_unique_ids:
-            yield ValidationError("Non-unique ID Values (first 3 shown):  {}".format(", ".join(list(non_unique_ids)[:3])))
+            yield ValidationError("Non-unique ID Values (first 3 shown):  {}".format(", ".join(str(x) for x in list(non_unique_ids)[:3])))
 
 
 def required_draft4(validator, required, instance, schema):

cove/test_hypothesis.py

@@ -0,0 +1,76 @@
+from cove.lib.ocds import get_releases_aggregates
+from cove.lib.threesixtygiving import get_grants_aggregates
+from hypothesis import given, assume, strategies as st, example, settings
+from cove.input.models import SuppliedData
+from django.core.files.base import ContentFile
+import pytest
+import json
+
+
+"""
+## Suggested testing patterns (from CamPUG talk)
+* simple fuzzing
+* round trip
+* invariants and idempotents
+* test oracle
+"""
+
+general_json = st.recursive(st.floats() | st.integers() | st.booleans() | st.text() | st.none(),
+    lambda children: st.lists(children) | st.dictionaries(st.text(), children))
+
+
+@pytest.mark.xfail
+@given(general_json)
+def test_get_grants_aggregates(json_data):
+    get_grants_aggregates(json_data)
+
+
+@given(general_json)
+def test_get_releases_aggregates(json_data):
+    get_releases_aggregates(json_data)
+
+
+@given(general_json)
+def test_get_grants_aggregates_dict(json_data):
+    assume(type(json_data) is dict)
+    get_grants_aggregates(json_data)
+
+
+@given(general_json)
+def test_get_releases_aggregates_dict(json_data):
+    assume(type(json_data) is dict)
+    get_releases_aggregates(json_data)
+
+
+@pytest.mark.xfail
+@pytest.mark.django_db
+@pytest.mark.parametrize('current_app', ['cove-ocds'])  # , 'cove-360'])
+@given(
+    general_json |
+    st.fixed_dictionaries({'releases': general_json})
+    )
+def test_explore_page(client, current_app, json_data):
+    data = SuppliedData.objects.create()
+    data.original_file.save('test.json', ContentFile(json.dumps(json_data)))
+    data.current_app = current_app
+    resp = client.get(data.get_absolute_url())
+    assert resp.status_code == 200
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize('current_app', ['cove-ocds'])  # , 'cove-360'])
+@given(general_json)
+@example(1)
+@settings(max_examples=50)
+def test_explore_page_duplicate_ids(client, current_app, json_data):
+    duplicate_id_releases = {
+        'releases': [
+            {'id': json_data},
+            {'id': json_data}
+        ]
+    }
+    data = SuppliedData.objects.create()
+    data.original_file.save('test.json', ContentFile(json.dumps(duplicate_id_releases)))
+    data.current_app = current_app
+    resp = client.get(data.get_absolute_url())
+    assert resp.status_code == 200

requirements_dev.in

@@ -15,4 +15,5 @@ transifex-client
 libsass
 Sphinx
 recommonmark
+hypothesis
 

requirements_dev.txt

@@ -27,6 +27,7 @@ transifex-client==0.12.1
 libsass==0.11.1
 Sphinx==1.4.5
 recommonmark==0.4.0
+hypothesis==3.5.3
 
 ## The following requirements were added by pip freeze:
 alabaster==0.7.8
@@ -54,6 +55,9 @@ pytz==2016.6.1
 schema==0.6.0
 six==1.10.0
 snowballstemmer==1.2.1
+spark-parser==1.4.0
 sqlparse==0.1.19
+uncompyle6==2.9.3
 urllib3==1.16
 Werkzeug==0.11.10
+xdis==3.2.3