Skip to content

Commit

Permalink
datastore: data_quality: rewrite_quality_data handle single failures
Browse files Browse the repository at this point in the history
If one publisher/source failed don't bail out of the whole process.
  • Loading branch information
michaelwood committed Feb 7, 2022
1 parent ec1bfcd commit 5b0a6c8
Showing 1 changed file with 24 additions and 12 deletions.
36 changes: 24 additions & 12 deletions datastore/data_quality/management/commands/rewrite_quality_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,21 @@ def handle(self, *args, **options):
)

def process_source_file(source_file):
grants_list = {
"grants": list(source_file.grant_set.values_list("data", flat=True))
}
source_file.quality, source_file.aggregate = quality_data.create(
grants_list
)
source_file.save()
try:
grants_list = {
"grants": list(source_file.grant_set.values_list("data", flat=True))
}
source_file.quality, source_file.aggregate = quality_data.create(
grants_list
)
source_file.save()
except Exception as e:
print(
"Could not create source file quality data for %s"
% str(source_file)
)
print(e)

connection.close()

with Pool(4) as process_pool:
Expand All @@ -48,11 +56,15 @@ def process_source_file(source_file):
def process_publishers(source_file):
publisher = source_file.get_publisher()

(
publisher.quality,
publisher.aggregate,
) = quality_data.create_publisher_stats(publisher)
publisher.save()
try:
(
publisher.quality,
publisher.aggregate,
) = quality_data.create_publisher_stats(publisher)
publisher.save()
except Exception as e:
print("Could not create publisher quality data for %s" % str(publisher))
print(e)
connection.close()

with Pool(4) as process_pool:
Expand Down

0 comments on commit 5b0a6c8

Please sign in to comment.