Improve error message for JSON parsing errors

This works by overriding the Flask Request's json
`on_loading_json_failed(e)` function so that it returns a descriptive
HTTP 400 rather than an empty one.

Note that we're trying to test against a specific exception string
defined by the JSON class - this is not stable (see
python/cpython@d4b0761)
- so I've made the message matcher use regular expressions rather than
exact string match. This required upgrading the hamcrest module to a
later version for `matches_regexp`

https://www.pivotaltracker.com/story/show/69911234

[Delivers #69911234]

backdrop/write/api.py

@@ -112,7 +112,7 @@ def write_by_group(data_group, data_type):
     _validate_auth(data_set_config)
 
     try:
-        data = listify_json(request.json)
+        data = listify_json(get_json_from_request(request))
         return _append_to_data_set(data_set_config, data)
 
     except (ParseError, ValidationError) as e:
@@ -137,7 +137,7 @@ def put_by_group_and_type(data_group, data_type):
     _validate_auth(data_set_config)
 
     try:
-        data = listify_json(request.json)
+        data = listify_json(get_json_from_request(request))
         if len(data) > 0:
             abort(400, 'Not implemented: you can only pass an empty JSON list')
 
@@ -158,7 +158,7 @@ def post_to_data_set(data_set_name):
     _validate_auth(data_set_config)
 
     try:
-        data = listify_json(request.json)
+        data = listify_json(get_json_from_request(request))
         return _append_to_data_set(
             data_set_config,
             data,
@@ -263,6 +263,15 @@ def _empty_data_set(data_set_config):
         message='{} now contains 0 records'.format(data_set_config.name))
 
 
+def get_json_from_request(request):
+    def json_error_handler(e):
+        app.logger.exception(e)
+        abort(400, 'Error parsing JSON: "{}"'.format(str(e)))
+
+    request.on_json_loading_failed = json_error_handler
+    return request.get_json()
+
+
 def listify_json(data):
     if data is None:
         raise ValidationError(

features/steps/read_api.py

@@ -3,6 +3,7 @@
 from behave import *
 from flask import json
 from hamcrest import *
+from hamcrest import matches_regexp
 from dateutil import parser
 import datetime
 import re
@@ -117,7 +118,7 @@ def step(context, header, value):
 @then('I should get back the message "{message}"')
 def step(context, message):
     data = json.loads(context.response.data)
-    assert_that(data["message"], is_(message))
+    assert_that(data["message"], matches_regexp(message))
 
 
 @then('I should get back a message: "{expected_message}"')

features/write_api/write_api.feature

@@ -71,4 +71,4 @@ Feature: the performance platform write api
           and I use the bearer token for the data_set
          when I POST to the specific path "/data/group/type"
          then I should get back a status of "400"
-         and I should get back the message "Error parsing JSON: "Expecting property name: line 1 column 1 (char 1)""
+         and I should get back the message "Error parsing JSON: .*""

requirements_for_tests.txt

@@ -6,7 +6,7 @@ coveralls
 mock==1.0.1
 nose==1.3.0
 pep8==1.4.5
-PyHamcrest==1.7.1
+PyHamcrest==1.8.0
 selenium==2.37.2
 splinter==0.5.4
 freezegun==0.1.11