Skip to content

Commit

Permalink
360 cove: quality checks in numbers
Browse files Browse the repository at this point in the history 
quality checks were in percentages.
#1200
  • Loading branch information
@BibianaC
BibianaC committed Mar 31, 2020
1 parent 853e1bf commit 002203f
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 26 deletions.
46 changes: 27 additions & 19 deletions cove_360/lib/threesixtygiving.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -285,13 +285,14 @@ def common_checks_360(context, upload_dir, json_data, schema_obj):
'validation_errors_grouped': group_validation_errors(context['validation_errors'], context['file_type'], openpyxl_workbook)
})

for test_classes_type in ['quality_accuracy', 'usefulness']:
for test_class_type in ['quality_accuracy', 'usefulness']:
extra_checks = run_extra_checks(
json_data, cell_source_map, TEST_CLASSES[test_classes_type], ignore_errors=True, return_on_error=None)
json_data, cell_source_map, TEST_CLASSES[test_class_type], ignore_errors=True, return_on_error=None)

context.update({
'{}_errored'.format(test_classes_type): extra_checks is None,
'{}_checks'.format(test_classes_type): extra_checks,
'{}_checks_count'.format(test_classes_type): (len(extra_checks) if extra_checks else 0) + (1 if context['data_only'] else 0)
'{}_errored'.format(test_class_type): extra_checks is None,
'{}_checks'.format(test_class_type): extra_checks,
'{}_checks_count'.format(test_class_type): (len(extra_checks) if extra_checks else 0) + (1 if context['data_only'] else 0)
})

return context
Expand Down Expand Up @@ -375,30 +376,36 @@ def produce_message(self):
'message': self.message
}

def get_heading_count(self):
def get_heading_count(self, test_class_type):
if test_class_type == 'quality_accuracy':
return self.count

if self.grants_count == 1 and self.count == 1:
self.grants_percentage = 100
return '1'

heading_percentage = '{:.0%}'.format(self.count / self.grants_count)
self.grants_percentage = int(heading_percentage[:-1])

return '{} of'.format(heading_percentage)

def format_heading_count(self, message, verb='have'):
def format_heading_count(self, message, test_class_type=None, verb='have'):
"""Build a string with count of grants plus message
The grant count phrase for the test is pluralized and
prepended to message, eg: 1 grant has + message,
2 grants have + message or 3 grants contain + message.
"""
noun = 'grant' if self.grants_count == 1 else 'grants'
count = self.count if test_class_type == 'quality_accuracy' else self.grants_count
noun = 'grant' if count == 1 else 'grants'
if verb == 'have':
verb = 'has' if self.grants_count == 1 else verb
verb = 'has' if count == 1 else verb
elif verb == 'do':
verb = 'does' if self.grants_count == 1 else verb
verb = 'does' if count == 1 else verb
else:
# Naively!
verb = verb + 's' if self.grants_count == 1 else verb
return '{} {} {} {}'.format(self.get_heading_count(), noun, verb, message)
verb = verb + 's' if self.count == 1 else verb
return '{} {} {} {}'.format(self.get_heading_count(test_class_type), noun, verb, message)


class ZeroAmountTest(AdditionalTest):
Expand Down Expand Up @@ -429,7 +436,7 @@ def process(self, grant, path_prefix):
except KeyError:
pass

self.heading = mark_safe(self.format_heading_count(self.check_text['heading']))
self.heading = mark_safe(self.format_heading_count(self.check_text['heading'], test_class_type='quality_accuracy'))
self.message = self.check_text['message'][self.grants_percentage]


Expand Down Expand Up @@ -523,7 +530,7 @@ def process(self, grant, path_prefix):
except KeyError:
pass

self.heading = self.format_heading_count(self.check_text['heading'])
self.heading = self.format_heading_count(self.check_text['heading'], test_class_type='quality_accuracy')
self.message = self.check_text['message'][self.grants_percentage]


Expand Down Expand Up @@ -560,7 +567,7 @@ def process(self, grant, path_prefix):
except KeyError:
pass

self.heading = mark_safe(self.format_heading_count(self.check_text['heading']))
self.heading = mark_safe(self.format_heading_count(self.check_text['heading'], test_class_type='quality_accuracy'))
self.message = self.check_text['message'][self.grants_percentage]


Expand Down Expand Up @@ -599,7 +606,7 @@ def process(self, grant, path_prefix):
except KeyError:
pass

self.heading = mark_safe(self.format_heading_count(self.check_text['heading']))
self.heading = mark_safe(self.format_heading_count(self.check_text['heading'], test_class_type='quality_accuracy'))
self.message = self.check_text['message'][self.grants_percentage]


Expand Down Expand Up @@ -638,7 +645,7 @@ def process(self, grant, path_prefix):
except KeyError:
pass

self.heading = mark_safe(self.format_heading_count(self.check_text['heading']))
self.heading = mark_safe(self.format_heading_count(self.check_text['heading'], test_class_type='quality_accuracy'))
self.message = mark_safe(self.check_text['message'][self.grants_percentage])


Expand Down Expand Up @@ -781,7 +788,7 @@ def process(self, grant, path_prefix):
self.json_locations.append(path_prefix + key)
self.count += 1

self.heading = self.format_heading_count(self.check_text['heading'], verb='contain')
self.heading = self.format_heading_count(self.check_text['heading'], test_class_type='quality_accuracy', verb='contain')
self.message = self.check_text['message'][self.grants_percentage]


Expand Down Expand Up @@ -911,7 +918,7 @@ def process(self, grant, path_prefix):
self.json_locations.append(id_location)
self.count += 1

self.heading = self.format_heading_count(self.check_text['heading'])
self.heading = self.format_heading_count(self.check_text['heading'], test_class_type='quality_accuracy')
self.message = self.check_text['message'][self.grants_percentage]


Expand Down Expand Up @@ -997,6 +1004,7 @@ def process(self, grant, path_prefix):
def run_extra_checks(json_data, cell_source_map, test_classes):
if 'grants' not in json_data:
return []

test_instances = [test_cls(grants=json_data['grants']) for test_cls in test_classes]

for num, grant in enumerate(json_data['grants']):
Expand Down
14 changes: 7 additions & 7 deletions cove_360/tests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,15 +293,15 @@


QUALITY_ACCURACY_CHECKS_RESULTS = [
({'heading': "33% of grants have a value of £0",
({'heading': "1 grant has a value of £0",
'message': ("It’s worth taking a look at these grants and deciding if "
"they should be published. It’s unusual to have grants of £0, but "
"there may be a reasonable explanation. Additional information "
"on why these grants are £0 might be useful to anyone using the data, "
"so consider adding an explanation to the description of the grant.")},
['grants/0/amountAwarded'],
[{'sheet': 'grants', 'letter': 'Q', 'row_number': 2, 'header': 'Amount Awarded'}]),
({'heading': ("33% of grants have a <span class=\"highlight-background-text\">Funding Org:Identifier</span> that "
({'heading': ("1 grant has a <span class=\"highlight-background-text\">Funding Org:Identifier</span> that "
"does not draw from a recognised register."),
'message': ("Using external identifiers (such as a charity or company number) helps "
"people using your data to match it up against other data - for example "
Expand All @@ -311,7 +311,7 @@
"then you can ignore this notice.")},
['grants/0/fundingOrganization/0/id'],
[{'sheet': 'grants', 'letter': 'V', 'row_number': 2, 'header': 'Funding Org:Identifier'}]),
({'heading': ("33% of grants have a <span class=\"highlight-background-text\">Recipient Org:Identifier</span> that "
({'heading': ("1 grant has a <span class=\"highlight-background-text\">Recipient Org:Identifier</span> that "
"does not draw from a recognised register."),
'message': ("Using external identifiers (such as a charity or company number) helps "
"people using your data to match it up against other data - for example "
Expand All @@ -321,14 +321,14 @@
"then you can ignore this notice.")},
['grants/1/recipientOrganization/0/id'],
[{'sheet': 'grants', 'letter': 'J', 'row_number': 3, 'header': 'Recipient Org:Identifier'}]),
({'heading': ("33% of grants have a value provided in the "
({'heading': ("1 grant has a value provided in the "
"<span class=\"highlight-background-text\">Recipient Org: Charity Number</span> column "
"that doesn’t look like a charity number"),
'message': ("Common causes of this are missing leading digits, typos or incorrect "
"values being entered into this field.")},
['grants/0/recipientOrganization/0/charityNumber'],
[{'sheet': 'grants', 'letter': 'M', 'row_number': 2, 'header': 'Recipient Org:Charity Number'}]),
({'heading': ("33% of grants have a value provided in the "
({'heading': ("1 grant has a value provided in the "
"<span class=\"highlight-background-text\">Recipient Org: Company Number</span> column "
"that doesn’t look like a company number"),
'message': ("Common causes of this are missing leading digits, typos or incorrect values "
Expand All @@ -338,7 +338,7 @@
"company numbers online at <a href=\"https://beta.companieshouse.gov.uk/\">Companies House</a>.")},
['grants/0/recipientOrganization/0/companyNumber'],
[{'sheet': 'grants', 'letter': 'L', 'row_number': 2, 'header': 'Recipient Org:Company Number'}]),
({'heading': "67% of grants have funder or recipient organisation IDs that might not be valid",
({'heading': "2 grants have funder or recipient organisation IDs that might not be valid",
'message': ("The IDs might not be valid for the registration agency that they refer to "
"- for example, a 'GB-CHC' ID that contains an invalid charity number. Common "
"causes of this are missing leading digits, typos or incorrect values being "
Expand All @@ -355,7 +355,7 @@
[{'sheet': 'grants', 'letter': 'V', 'row_number': 2, 'header': 'Funding Org:Identifier'},
{'sheet': 'grants', 'letter': 'V', 'row_number': 3, 'header': 'Funding Org:Identifier'},
{'sheet': 'grants', 'letter': 'V', 'row_number': 4, 'header': 'Funding Org:Identifier'}]),
({'heading': "67% of grants contain text that looks like an email address",
({'heading': "2 grants contain text that looks like an email address",
'message': ("Your data may contain an email address (or something that looks like one), "
"which can constitute personal data. The use and distribution of personal data "
"is restricted by the Data Protection Act. You should ensure that any personal "
Expand Down

0 comments on commit 002203f

Please sign in to comment.