Skip to content

Commit

Permalink
Invalid Footprint DQ duplicate rules are not recreated
Browse files Browse the repository at this point in the history
  • Loading branch information
Adrian Lara committed Mar 25, 2019
1 parent 8525fa4 commit b94fea8
Show file tree
Hide file tree
Showing 5 changed files with 29 additions and 1 deletion.
2 changes: 1 addition & 1 deletion seed/data_importer/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,7 @@ def _store_raw_footprint_and_create_rule(footprint_details, table, org, import_f
}

dq, _created = DataQualityCheck.objects.get_or_create(organization=org.id)
dq.add_rule(rule)
dq.add_rule_if_new(rule)


def _map_data_create_tasks(import_file_id, progress_key):
Expand Down
Binary file not shown.
Binary file not shown.
17 changes: 17 additions & 0 deletions seed/data_importer/tests/integration/test_footprints_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ def test_demo_v2(self):
# Make sure that new DQ rules have been added and apply to the states with (Invalid Footprints)
tdq = DataQualityCheck.retrieve(self.org.id)
tdq.check_data('TaxLotState', [tax_lot_1, tax_lot_2, tax_lot_3])
initial_tdq_rules_count = tdq.rules.count()
self.assertEqual(tdq.results.get(tax_lot_1.id, None), None)
self.assertEqual(tdq.results[tax_lot_2.id]['data_quality_results'][0]['detailed_message'], "'(( -121.927490629756 37.3...' is not a valid geometry")
self.assertEqual(tdq.results[tax_lot_3.id]['data_quality_results'][0]['detailed_message'], "'' is not a valid geometry")
Expand All @@ -180,3 +181,19 @@ def test_demo_v2(self):
self.assertEqual(pdq.results.get(property_1.id, None), None)
self.assertEqual(pdq.results[property_2.id]['data_quality_results'][0]['detailed_message'], "'{}' is not a valid geometry".format(invalid_property_footprint_string))
self.assertEqual(pdq.results[property_3.id]['data_quality_results'][0]['detailed_message'], "'123' is not a valid geometry")

# Run new import, and check that duplicate rules are not created
new_import_file_tax_lot = ImportFile.objects.create(
import_record=self.import_record_tax_lot, cycle=self.cycle
)
tax_lot_filename = getattr(self, 'filename', 'example-data-taxlots-1-invalid-footprint.xlsx')
filepath = osp.join(osp.dirname(__file__), '..', 'data', tax_lot_filename)
new_import_file_tax_lot.file = SimpleUploadedFile(
name=tax_lot_filename,
content=open(filepath, 'rb').read()
)
new_import_file_tax_lot.save()

tasks.save_raw_data(new_import_file_tax_lot.pk)
updated_tdq = DataQualityCheck.retrieve(self.org.id)
self.assertEqual(updated_tdq.rules.count(), initial_tdq_rules_count)
11 changes: 11 additions & 0 deletions seed/models/data_quality.py
Original file line number Diff line number Diff line change
Expand Up @@ -785,6 +785,17 @@ def add_rule(self, rule):

self.rules.add(r)

def add_rule_if_new(self, rule):
"""
Add a new rule to the Data Quality Checks only if rule does not exist
:param rule: dict to be added as a new rule
:return: None
"""
rule_exists = self.rules.get_queryset().filter(**rule).exists()
if not rule_exists:
self.add_rule(rule)

def add_result_string_error(self, row_id, rule, display_name, value):
self.results[row_id]['data_quality_results'].append(
{
Expand Down

0 comments on commit b94fea8

Please sign in to comment.