From cacba118e0d1e4514fdae9880f67d98f7a8574a8 Mon Sep 17 00:00:00 2001 From: Evildoor Date: Thu, 18 Apr 2019 15:27:02 +0200 Subject: [PATCH] Remove batching of inconsistent records. Printing all discovered inconsistent records to stdout as a batch contradicts with various things, such as pyDKB's file mode and the possibility of controlling the workflow with Apache Kafka. Create an output message with _id and _type for each inconsistent record. Still exit with code 1 if at least one inconsistent record was found, 0 otherwise. --- Utils/Dataflow/071_esConsistency/consistency.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Utils/Dataflow/071_esConsistency/consistency.py b/Utils/Dataflow/071_esConsistency/consistency.py index 641b581e6..e7f8214f1 100755 --- a/Utils/Dataflow/071_esConsistency/consistency.py +++ b/Utils/Dataflow/071_esConsistency/consistency.py @@ -42,7 +42,7 @@ def log(msg, prefix='DEBUG'): INDEX = None -FOUND_DIFF = [] +FOUND_DIFF = False def load_config(fname): @@ -166,8 +166,10 @@ def process(stage, message): if data != es_data: log('Document (%s, %d) differs between Oracle and ES: Oracle:%s ES:%s' % (_type, _id, data, es_data), 'WARN') + out_message = JSONMessage({'_type': _type, '_id': _id}) + stage.output(out_message) global FOUND_DIFF - FOUND_DIFF.append((_type, _id)) + FOUND_DIFF = True else: log('Document (%s, %d) is up to date in ES' % (_type, _id), 'INFO') @@ -218,8 +220,6 @@ def main(args): if exit_code == 0 and FOUND_DIFF: exit_code = 1 - print FOUND_DIFF - exit(exit_code)