01_Fraud_Detection_Demo_Params_ExecuteScript.json

{"flowContents":{"identifier":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172","instanceIdentifier":"12fed286-0182-1000-0000-00000271ff2c","name":"01_Fraud_Detection_Demo_srm_merged","comments":"","position":{"x":368.0,"y":240.0},"processGroups":[],"remoteProcessGroups":[],"processors":[{"identifier":"4de286d1-87cf-3040-84a3-ba3d1c9587cf","instanceIdentifier":"4793e3d6-9439-3795-8bd2-1bed8c7b1a7d","name":"Set Schema Name","comments":"","position":{"x":696.0,"y":256.0},"type":"org.apache.nifi.processors.attributes.UpdateAttribute","bundle":{"group":"org.apache.nifi","artifact":"nifi-update-attribute-nar","version":"1.16.0.2.2.5.1-1"},"properties":{"Delete Attributes Expression":null,"Store State":"Do not store state","canonical-value-lookup-cache-size":"100","schema.name":"fintxn","Stateful Variables Initial Value":null},"propertyDescriptors":{"Delete Attributes Expression":{"name":"Delete Attributes Expression","displayName":"Delete Attributes Expression","identifiesControllerService":false,"sensitive":false},"Store State":{"name":"Store State","displayName":"Store State","identifiesControllerService":false,"sensitive":false},"canonical-value-lookup-cache-size":{"name":"canonical-value-lookup-cache-size","displayName":"Cache Value Lookup Cache Size","identifiesControllerService":false,"sensitive":false},"schema.name":{"name":"schema.name","displayName":"schema.name","identifiesControllerService":false,"sensitive":false},"Stateful Variables Initial Value":{"name":"Stateful Variables Initial Value","displayName":"Stateful Variables Initial Value","identifiesControllerService":false,"sensitive":false}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172"},{"identifier":"fe0ef94c-a849-37fe-b838-8f8608383e58","instanceIdentifier":"2f4cd004-947a-37fe-9ca9-bc6aae5f4ecb","name":"CreateTransactions","comments":"","position":{"x":696.0,"y":64.0},"type":"org.apache.nifi.processors.script.ExecuteScript","bundle":{"group":"org.apache.nifi","artifact":"nifi-scripting-nar","version":"1.16.0.2.2.5.1-1"},"properties":{"Script File":null,"Script Engine":"python","Script Body":"#!/usr/bin/python\n\"\"\" \n  The implementation of the synthetic financial transaction stream,\n  used in the main script of gess.\n\n@author: Michael Hausenblas, http://mhausenblas.info/#i\n@since: 2013-11-08\n@status: init\n\"\"\"\nimport sys\nimport os\nimport socket\nimport logging\nimport string\nimport datetime\nimport random\nimport uuid\nimport csv\nimport json\nfrom time import sleep\nfrom random import randint\nimport os\nimport random\nimport math\nfrom random import uniform\nimport time\nfrom time import sleep\n\nimport java.io\nfrom org.apache.commons.io import IOUtils\nfrom java.nio.charset import StandardCharsets\nfrom org.apache.nifi.processor.io import StreamCallback\nimport traceback\nfrom org.apache.nifi.processor.io import OutputStreamCallback\nfrom org.python.core.util import StringUtil\n\n\n#The target host Ip address need to be updated with the current UDP Server  IP address.\nTARGET_HOST = \"10.10.242.34\"\nUDP_PORT = 6900\nDELAY = 1\n\n# defines the sampling interval (in seconds) for reporting runtime statistics\nSAMPLE_INTERVAL = 10\n\n# lower range for randomly emitted frauds (min. tick between trans)\nFRAUD_TICK_MIN = 5\n\n# upper range for randomly emitted frauds (max. tick between trans)\nFRAUD_TICK_MAX = 15\n\n\n# Add some data = Amounts and Cities.\nAMOUNTS = [20, 50, 100, 200, 300, 400,500]\nCITIES = [                                                                                                                                                                                                                                                     \n    {\"lat\": 48.8534, \"lon\": 2.3488, \"city\": \"Paris\"},                                                                                                                                                                                                    \n    {\"lat\": 43.2961743, \"lon\": 5.3699525, \"city\": \"Marseille\"},                                                                                                                                                                                                 \n    {\"lat\": 45.7578137, \"lon\": 4.8320114, \"city\": \"Lyon\"},                                                                                                                                                                                                      \n    {\"lat\": 50.6365654, \"lon\": 3.0635282, \"city\": \"Lille\"},\n    {\"lat\": 44.841225, \"lon\": -0.5800364, \"city\": \"Bordeaux\"}\n]        \n\n# Define geo functions\ndef create_random_point(x0, y0, distance):\n    r = distance/111300\n    u = random.random()\n    v = random.random()\n    w = r * math.sqrt(u)\n    t = 2 * math.pi * v\n    x = w * math.cos(t)\n    x1 = x / math.cos(y0)\n    y = w * math.sin(t)\n    return (x0+x1, y0 +y)\n\ndef create_geopoint(lat, lon):\n    return create_random_point(lat, lon, 50000)\n\ndef get_latlon():                                                                                                                                                                                                                                              \n    geo = random.choice(CITIES)\n    return create_geopoint(geo['lat'], geo['lon']),geo['city']        \n\n\n# creates a single financial transaction  using the following\n  # format:\n#    {\n#       'ts': '2013-11-08T10:58:19.668225',\n#       'account_id': 'a335',\n#       'transaction_id': '636adacc-49d2-11e3-a3d1-a820664821e3'\n#       'amount': 100,\n#       'lat': '36.7220096',\n#       'lon': '-4.4186772'\n#     }\n\ndef create_fintran():\n \n    latlon,city = get_latlon()\n    tsbis=(datetime.datetime.now()).strftime(\"%Y-%m-%d %H:%M:%S \")\n    date = str(datetime.datetime.strptime(tsbis, \"%Y-%m-%d %H:%M:%S \"))\n    fintran = {\n      'ts': date,\n      'account_id' : str(random.randint(1, 1000)),\n      'transaction_id' : str(uuid.uuid1()),\n      'amount' : random.randrange(1,2000),  \n      'lat' : latlon[0],\n      'lon' : latlon[1]\n    }    \n\n    return (fintran)\n\n# creates a single fraudulent financial transaction\n  # based on an existing transaction, using the following format:\n  # {\n  #   'ts': '2013-11-08T12:28:39.466325',\n  #    'account_id': 'a335',\n  #   'transaction_id': 'xxx636adacc-49d2-11e3-a3d1-a820664821e3'\n  #   'amount': 200,\n  #   'lat': '39.5655472',\n  #   'lon': '-0.530058'\n  # }\n  # Note: the fraudulent transaction will have the same account ID as\n  #       the original transaction but different location and ammount.\n\ndef create_fraudtran(fintran):\n   \n    latlon,city = get_latlon()\n    tsbis = str((datetime.datetime.now() - datetime.timedelta(seconds=random.randint(60,600))).strftime(\"%Y-%m-%d %H:%M:%S \"))\n    \n    fraudtran = {\n      'ts' : tsbis,\n      'account_id' : fintran['account_id'],\n      'transaction_id' : 'xxx' + str(fintran['transaction_id']),\n      'amount' : random.randrange(1,2000),      \n      'lat' : latlon[0],\n      'lon' : latlon[1]\n    }    \n    return (fraudtran)\n\n\ndef send_fintran(out_socket, fintran):\n    out_socket.sendto(str.encode(fintran), (TARGET_HOST, UDP_PORT))\n\nclass WriteContentCallback(OutputStreamCallback):\n    def __init__(self, content):\n        self.content_text = content\n\n    def process(self, outputStream):\n        try:\n            outputStream.write(StringUtil.toBytes(self.content_text))\n        except:\n            traceback.print_exc(file=sys.stdout)\n            raise\n\n# Main programm \n\n#out_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # use UDP\nticks = 0 # ticks (virtual time basis for emits)\nfraud_tick = random.randint(FRAUD_TICK_MIN, FRAUD_TICK_MAX) \n\n# get the flowfile\noriginalFlowFile = session.get()\nsession.remove(originalFlowFile)\n# All processing code starts at this indent\nwhile ticks < 20:\n ticks += 1\n fintran = create_fintran()   \n fintransaction =  json.dumps(fintran)\n #send_fintran(out_socket, json.dumps(fintran))\n #print(fintransaction)\n flowFile = session.create()\n flowFile = session.write(flowFile, WriteContentCallback(fintransaction))\n session.transfer(flowFile, REL_SUCCESS)\n session.commit()\n\n sleep(DELAY)\n if ticks > fraud_tick:\n    fraudtran = create_fraudtran(fintran)\n    fraudfintransaction=json.dumps(fraudtran)\n    #send_fintran(out_socket, json.dumps(fraudtran))\n    #print(fraudfintransaction)\n    flowFile2 = session.create()\n    flowFile2 = session.write(flowFile2, WriteContentCallback(fraudfintransaction))\n    session.transfer(flowFile2, REL_SUCCESS)\n    session.commit()\n    fraud_tick = random.randint(FRAUD_TICK_MIN, FRAUD_TICK_MAX)\n","Module Directory":null},"propertyDescriptors":{"Script File":{"name":"Script File","displayName":"Script File","identifiesControllerService":false,"sensitive":false,"resourceDefinition":{"cardinality":"SINGLE","resourceTypes":["FILE"]}},"Script Engine":{"name":"Script Engine","displayName":"Script Engine","identifiesControllerService":false,"sensitive":false},"Script Body":{"name":"Script Body","displayName":"Script Body","identifiesControllerService":false,"sensitive":false},"Module Directory":{"name":"Module Directory","displayName":"Module Directory","identifiesControllerService":false,"sensitive":false,"resourceDefinition":{"cardinality":"MULTIPLE","resourceTypes":["DIRECTORY","FILE"]}}},"style":{"background-color":"#ffffff"},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["failure"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172"},{"identifier":"59dedf23-be3e-33e5-87c0-4ef606fef905","instanceIdentifier":"8d45c963-791a-38ae-bb30-e63e9e15b857","name":"Publish to Kafka topic: txn1","comments":"","position":{"x":304.0,"y":464.0},"type":"org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6","bundle":{"group":"org.apache.nifi","artifact":"nifi-kafka-2-6-nar","version":"1.16.0.2.2.5.1-1"},"properties":{"compression.type":"none","attribute-name-regex":"schema.*","record-writer":"f5eda5fd-5bf0-3b5a-8d1c-a399b41f29d9","bootstrap.servers":"#{Kafka Brokers}","sasl.kerberos.principal":null,"sasl.kerberos.service.name":null,"kerberos-credentials-service":null,"partition":null,"sasl.mechanism":"PLAIN","max.block.ms":"10 sec","record-reader":"d8dcb534-91fd-36ca-a079-7c99b6a43dab","message-header-encoding":"UTF-8","kerberos-user-service":null,"transactional-id-prefix":null,"client.id":"nifi-txn-data","ack.wait.time":"10 secs","sasl.username":"#{CDP Workload User}","use-transactions":"false","acks":"0","security.protocol":"SASL_SSL","ssl.context.service":"b2790196-adee-3d53-ba8d-3a777ccb68be","max.request.size":"1 MB","sasl.token.auth":"false","message-key-field":null,"Failure Strategy":"Route to Failure","partitioner.class":"org.apache.kafka.clients.producer.internals.DefaultPartitioner","sasl.kerberos.keytab":null,"topic":"txn1","sasl.password":"#{CDP Workload User Password}"},"propertyDescriptors":{"compression.type":{"name":"compression.type","displayName":"Compression Type","identifiesControllerService":false,"sensitive":false},"attribute-name-regex":{"name":"attribute-name-regex","displayName":"Attributes to Send as Headers (Regex)","identifiesControllerService":false,"sensitive":false},"record-writer":{"name":"record-writer","displayName":"Record Writer","identifiesControllerService":true,"sensitive":false},"bootstrap.servers":{"name":"bootstrap.servers","displayName":"Kafka Brokers","identifiesControllerService":false,"sensitive":false},"sasl.kerberos.principal":{"name":"sasl.kerberos.principal","displayName":"Kerberos Principal","identifiesControllerService":false,"sensitive":false},"sasl.kerberos.service.name":{"name":"sasl.kerberos.service.name","displayName":"Kerberos Service Name","identifiesControllerService":false,"sensitive":false},"kerberos-credentials-service":{"name":"kerberos-credentials-service","displayName":"Kerberos Credentials Service","identifiesControllerService":true,"sensitive":false},"partition":{"name":"partition","displayName":"Partition","identifiesControllerService":false,"sensitive":false},"sasl.mechanism":{"name":"sasl.mechanism","displayName":"SASL Mechanism","identifiesControllerService":false,"sensitive":false},"max.block.ms":{"name":"max.block.ms","displayName":"Max Metadata Wait Time","identifiesControllerService":false,"sensitive":false},"record-reader":{"name":"record-reader","displayName":"Record Reader","identifiesControllerService":true,"sensitive":false},"message-header-encoding":{"name":"message-header-encoding","displayName":"Message Header Encoding","identifiesControllerService":false,"sensitive":false},"kerberos-user-service":{"name":"kerberos-user-service","displayName":"Kerberos User Service","identifiesControllerService":true,"sensitive":false},"transactional-id-prefix":{"name":"transactional-id-prefix","displayName":"Transactional Id Prefix","identifiesControllerService":false,"sensitive":false},"client.id":{"name":"client.id","displayName":"client.id","identifiesControllerService":false,"sensitive":false},"ack.wait.time":{"name":"ack.wait.time","displayName":"Acknowledgment Wait Time","identifiesControllerService":false,"sensitive":false},"sasl.username":{"name":"sasl.username","displayName":"Username","identifiesControllerService":false,"sensitive":false},"use-transactions":{"name":"use-transactions","displayName":"Use Transactions","identifiesControllerService":false,"sensitive":false},"acks":{"name":"acks","displayName":"Delivery Guarantee","identifiesControllerService":false,"sensitive":false},"security.protocol":{"name":"security.protocol","displayName":"Security Protocol","identifiesControllerService":false,"sensitive":false},"ssl.context.service":{"name":"ssl.context.service","displayName":"SSL Context Service","identifiesControllerService":true,"sensitive":false},"max.request.size":{"name":"max.request.size","displayName":"Max Request Size","identifiesControllerService":false,"sensitive":false},"sasl.token.auth":{"name":"sasl.token.auth","displayName":"Token Auth","identifiesControllerService":false,"sensitive":false},"message-key-field":{"name":"message-key-field","displayName":"Message Key Field","identifiesControllerService":false,"sensitive":false},"Failure Strategy":{"name":"Failure Strategy","displayName":"Failure Strategy","identifiesControllerService":false,"sensitive":false},"partitioner.class":{"name":"partitioner.class","displayName":"Partitioner class","identifiesControllerService":false,"sensitive":false},"sasl.kerberos.keytab":{"name":"sasl.kerberos.keytab","displayName":"Kerberos Keytab","identifiesControllerService":false,"sensitive":false,"resourceDefinition":{"cardinality":"SINGLE","resourceTypes":["FILE"]}},"topic":{"name":"topic","displayName":"Topic Name","identifiesControllerService":false,"sensitive":false},"sasl.password":{"name":"sasl.password","displayName":"Password","identifiesControllerService":false,"sensitive":true}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["success"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172"},{"identifier":"c39b1c89-62ff-34fe-bec8-659827c27796","instanceIdentifier":"30b10d93-08b2-32a1-8289-b337d61ddb95","name":"Publish to Kafka topic: txn2","comments":"","position":{"x":1016.0,"y":448.0},"type":"org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6","bundle":{"group":"org.apache.nifi","artifact":"nifi-kafka-2-6-nar","version":"1.16.0.2.2.5.1-1"},"properties":{"compression.type":"none","attribute-name-regex":"schema.*","record-writer":"f5eda5fd-5bf0-3b5a-8d1c-a399b41f29d9","bootstrap.servers":"#{Kafka Brokers}","sasl.kerberos.principal":null,"sasl.kerberos.service.name":null,"kerberos-credentials-service":null,"partition":null,"sasl.mechanism":"PLAIN","max.block.ms":"10 sec","record-reader":"d8dcb534-91fd-36ca-a079-7c99b6a43dab","message-header-encoding":"UTF-8","kerberos-user-service":null,"transactional-id-prefix":null,"client.id":"nifi-txn-data","ack.wait.time":"10 secs","sasl.username":"#{CDP Workload User}","use-transactions":"false","acks":"0","security.protocol":"SASL_SSL","ssl.context.service":"b2790196-adee-3d53-ba8d-3a777ccb68be","max.request.size":"1 MB","sasl.token.auth":"false","message-key-field":null,"Failure Strategy":"Route to Failure","partitioner.class":"org.apache.kafka.clients.producer.internals.DefaultPartitioner","sasl.kerberos.keytab":null,"topic":"txn2","sasl.password":"#{CDP Workload User Password}"},"propertyDescriptors":{"compression.type":{"name":"compression.type","displayName":"Compression Type","identifiesControllerService":false,"sensitive":false},"attribute-name-regex":{"name":"attribute-name-regex","displayName":"Attributes to Send as Headers (Regex)","identifiesControllerService":false,"sensitive":false},"record-writer":{"name":"record-writer","displayName":"Record Writer","identifiesControllerService":true,"sensitive":false},"bootstrap.servers":{"name":"bootstrap.servers","displayName":"Kafka Brokers","identifiesControllerService":false,"sensitive":false},"sasl.kerberos.principal":{"name":"sasl.kerberos.principal","displayName":"Kerberos Principal","identifiesControllerService":false,"sensitive":false},"sasl.kerberos.service.name":{"name":"sasl.kerberos.service.name","displayName":"Kerberos Service Name","identifiesControllerService":false,"sensitive":false},"kerberos-credentials-service":{"name":"kerberos-credentials-service","displayName":"Kerberos Credentials Service","identifiesControllerService":true,"sensitive":false},"partition":{"name":"partition","displayName":"Partition","identifiesControllerService":false,"sensitive":false},"sasl.mechanism":{"name":"sasl.mechanism","displayName":"SASL Mechanism","identifiesControllerService":false,"sensitive":false},"max.block.ms":{"name":"max.block.ms","displayName":"Max Metadata Wait Time","identifiesControllerService":false,"sensitive":false},"record-reader":{"name":"record-reader","displayName":"Record Reader","identifiesControllerService":true,"sensitive":false},"message-header-encoding":{"name":"message-header-encoding","displayName":"Message Header Encoding","identifiesControllerService":false,"sensitive":false},"kerberos-user-service":{"name":"kerberos-user-service","displayName":"Kerberos User Service","identifiesControllerService":true,"sensitive":false},"transactional-id-prefix":{"name":"transactional-id-prefix","displayName":"Transactional Id Prefix","identifiesControllerService":false,"sensitive":false},"client.id":{"name":"client.id","displayName":"client.id","identifiesControllerService":false,"sensitive":false},"ack.wait.time":{"name":"ack.wait.time","displayName":"Acknowledgment Wait Time","identifiesControllerService":false,"sensitive":false},"sasl.username":{"name":"sasl.username","displayName":"Username","identifiesControllerService":false,"sensitive":false},"use-transactions":{"name":"use-transactions","displayName":"Use Transactions","identifiesControllerService":false,"sensitive":false},"acks":{"name":"acks","displayName":"Delivery Guarantee","identifiesControllerService":false,"sensitive":false},"security.protocol":{"name":"security.protocol","displayName":"Security Protocol","identifiesControllerService":false,"sensitive":false},"ssl.context.service":{"name":"ssl.context.service","displayName":"SSL Context Service","identifiesControllerService":true,"sensitive":false},"max.request.size":{"name":"max.request.size","displayName":"Max Request Size","identifiesControllerService":false,"sensitive":false},"sasl.token.auth":{"name":"sasl.token.auth","displayName":"Token Auth","identifiesControllerService":false,"sensitive":false},"message-key-field":{"name":"message-key-field","displayName":"Message Key Field","identifiesControllerService":false,"sensitive":false},"Failure Strategy":{"name":"Failure Strategy","displayName":"Failure Strategy","identifiesControllerService":false,"sensitive":false},"partitioner.class":{"name":"partitioner.class","displayName":"Partitioner class","identifiesControllerService":false,"sensitive":false},"sasl.kerberos.keytab":{"name":"sasl.kerberos.keytab","displayName":"Kerberos Keytab","identifiesControllerService":false,"sensitive":false,"resourceDefinition":{"cardinality":"SINGLE","resourceTypes":["FILE"]}},"topic":{"name":"topic","displayName":"Topic Name","identifiesControllerService":false,"sensitive":false},"sasl.password":{"name":"sasl.password","displayName":"Password","identifiesControllerService":false,"sensitive":true}},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["success"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172"},{"identifier":"60aa2887-7067-3a9e-aaab-e39abd7d1080","instanceIdentifier":"3ef8ee09-47f3-38a3-be99-02735469a122","name":"StartGenerateTransactions","comments":"","position":{"x":712.0,"y":-120.0},"type":"org.apache.nifi.processors.standard.GenerateFlowFile","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"1.16.0.2.2.5.1-1"},"properties":{"character-set":"UTF-8","File Size":"0B","mime-type":null,"generate-ff-custom-text":null,"Batch Size":"1","Unique FlowFiles":"false","Data Format":"Text"},"propertyDescriptors":{"character-set":{"name":"character-set","displayName":"Character Set","identifiesControllerService":false,"sensitive":false},"File Size":{"name":"File Size","displayName":"File Size","identifiesControllerService":false,"sensitive":false},"mime-type":{"name":"mime-type","displayName":"Mime Type","identifiesControllerService":false,"sensitive":false},"generate-ff-custom-text":{"name":"generate-ff-custom-text","displayName":"Custom Text","identifiesControllerService":false,"sensitive":false},"Batch Size":{"name":"Batch Size","displayName":"Batch Size","identifiesControllerService":false,"sensitive":false},"Unique FlowFiles":{"name":"Unique FlowFiles","displayName":"Unique FlowFiles","identifiesControllerService":false,"sensitive":false},"Data Format":{"name":"Data Format","displayName":"Data Format","identifiesControllerService":false,"sensitive":false}},"style":{},"schedulingPeriod":"30 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"PRIMARY","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172"}],"inputPorts":[],"outputPorts":[],"connections":[{"identifier":"dba606a5-f111-37a2-9fb0-f65d10b82184","instanceIdentifier":"8ce5599e-1a6e-38d8-b03f-40dd26ddbc3f","name":"","source":{"id":"4de286d1-87cf-3040-84a3-ba3d1c9587cf","type":"PROCESSOR","groupId":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172","name":"Set Schema Name","comments":"","instanceIdentifier":"4793e3d6-9439-3795-8bd2-1bed8c7b1a7d"},"destination":{"id":"c39b1c89-62ff-34fe-bec8-659827c27796","type":"PROCESSOR","groupId":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172","name":"Publish to Kafka topic: txn2","comments":"","instanceIdentifier":"30b10d93-08b2-32a1-8289-b337d61ddb95"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172"},{"identifier":"a2ab077e-f037-32e3-8713-f6118e3497f5","instanceIdentifier":"846a40b6-00c3-33e4-98bb-4288db5075d2","name":"","source":{"id":"c39b1c89-62ff-34fe-bec8-659827c27796","type":"PROCESSOR","groupId":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172","name":"Publish to Kafka topic: txn2","comments":"","instanceIdentifier":"30b10d93-08b2-32a1-8289-b337d61ddb95"},"destination":{"id":"399e013a-3501-3ca8-a43b-4468065a4854","type":"FUNNEL","groupId":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172","name":"Funnel","comments":"","instanceIdentifier":"faf6a960-343a-3b39-9972-cc63442e1346"},"labelIndex":1,"zIndex":0,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172"},{"identifier":"83e6bff9-841f-3efd-abd3-eb1adb209a44","instanceIdentifier":"353e6912-12a6-3db6-9ad4-c868d306fdf0","name":"","source":{"id":"4de286d1-87cf-3040-84a3-ba3d1c9587cf","type":"PROCESSOR","groupId":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172","name":"Set Schema Name","comments":"","instanceIdentifier":"4793e3d6-9439-3795-8bd2-1bed8c7b1a7d"},"destination":{"id":"59dedf23-be3e-33e5-87c0-4ef606fef905","type":"PROCESSOR","groupId":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172","name":"Publish to Kafka topic: txn1","comments":"","instanceIdentifier":"8d45c963-791a-38ae-bb30-e63e9e15b857"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172"},{"identifier":"ecda2a62-f249-32f9-b3c7-1d956b120bec","instanceIdentifier":"84d06e1c-a1c1-3179-9fb1-a1a98da5bb74","name":"","source":{"id":"fe0ef94c-a849-37fe-b838-8f8608383e58","type":"PROCESSOR","groupId":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172","name":"CreateTransactions","comments":"","instanceIdentifier":"2f4cd004-947a-37fe-9ca9-bc6aae5f4ecb"},"destination":{"id":"4de286d1-87cf-3040-84a3-ba3d1c9587cf","type":"PROCESSOR","groupId":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172","name":"Set Schema Name","comments":"","instanceIdentifier":"4793e3d6-9439-3795-8bd2-1bed8c7b1a7d"},"labelIndex":0,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172"},{"identifier":"59607927-945e-3c71-a46f-e002bb4606b5","instanceIdentifier":"2e18d03b-4b43-33a9-99c5-8130332f8830","name":"","source":{"id":"60aa2887-7067-3a9e-aaab-e39abd7d1080","type":"PROCESSOR","groupId":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172","name":"StartGenerateTransactions","comments":"","instanceIdentifier":"3ef8ee09-47f3-38a3-be99-02735469a122"},"destination":{"id":"fe0ef94c-a849-37fe-b838-8f8608383e58","type":"PROCESSOR","groupId":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172","name":"CreateTransactions","comments":"","instanceIdentifier":"2f4cd004-947a-37fe-9ca9-bc6aae5f4ecb"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172"},{"identifier":"d15072a2-918c-3a5b-9624-09569a691306","instanceIdentifier":"bbc6c53d-d207-3a4b-836b-cfaedeceed68","name":"","source":{"id":"59dedf23-be3e-33e5-87c0-4ef606fef905","type":"PROCESSOR","groupId":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172","name":"Publish to Kafka topic: txn1","comments":"","instanceIdentifier":"8d45c963-791a-38ae-bb30-e63e9e15b857"},"destination":{"id":"399e013a-3501-3ca8-a43b-4468065a4854","type":"FUNNEL","groupId":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172","name":"Funnel","comments":"","instanceIdentifier":"faf6a960-343a-3b39-9972-cc63442e1346"},"labelIndex":1,"zIndex":0,"selectedRelationships":["failure"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","partitioningAttribute":"","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172"}],"labels":[{"identifier":"2f633700-9522-3303-bec0-71465670758a","instanceIdentifier":"179bc31e-0182-1000-ffff-ffffdf07b35b","position":{"x":1080.0,"y":56.0},"label":"need to clean up the original python lines/code which is not needed","zIndex":0,"width":768.0,"height":112.0,"style":{"background-color":"#f70f45","font-size":"24px"},"componentType":"LABEL","groupIdentifier":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172"}],"funnels":[{"identifier":"399e013a-3501-3ca8-a43b-4468065a4854","instanceIdentifier":"faf6a960-343a-3b39-9972-cc63442e1346","position":{"x":816.0,"y":680.0},"componentType":"FUNNEL","groupIdentifier":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172"}],"controllerServices":[{"identifier":"f5eda5fd-5bf0-3b5a-8d1c-a399b41f29d9","instanceIdentifier":"30b74b33-4661-39b3-8247-d410c2a00e28","name":"JsonRecordSetWriter","comments":"","type":"org.apache.nifi.json.JsonRecordSetWriter","bundle":{"group":"org.apache.nifi","artifact":"nifi-record-serialization-services-nar","version":"1.16.0.2.2.5.1-1"},"properties":{"schema-branch":null,"compression-level":"1","schema-cache":null,"Timestamp Format":null,"Date Format":null,"Pretty Print JSON":"false","compression-format":"none","Schema Write Strategy":"hwx-schema-ref-attributes","suppress-nulls":"never-suppress","output-grouping":"output-array","schema-name":"${schema.name}","schema-registry":"c602ce32-0f17-397f-87da-00d6c873454c","Time Format":null,"schema-access-strategy":"schema-name","schema-protocol-version":"1","schema-version":null,"schema-text":"${avro.schema}"},"propertyDescriptors":{"schema-branch":{"name":"schema-branch","displayName":"Schema Branch","identifiesControllerService":false,"sensitive":false},"compression-level":{"name":"compression-level","displayName":"Compression Level","identifiesControllerService":false,"sensitive":false},"schema-cache":{"name":"schema-cache","displayName":"Schema Cache","identifiesControllerService":true,"sensitive":false},"Timestamp Format":{"name":"Timestamp Format","displayName":"Timestamp Format","identifiesControllerService":false,"sensitive":false},"Date Format":{"name":"Date Format","displayName":"Date Format","identifiesControllerService":false,"sensitive":false},"Pretty Print JSON":{"name":"Pretty Print JSON","displayName":"Pretty Print JSON","identifiesControllerService":false,"sensitive":false},"compression-format":{"name":"compression-format","displayName":"Compression Format","identifiesControllerService":false,"sensitive":false},"Schema Write Strategy":{"name":"Schema Write Strategy","displayName":"Schema Write Strategy","identifiesControllerService":false,"sensitive":false},"suppress-nulls":{"name":"suppress-nulls","displayName":"Suppress Null Values","identifiesControllerService":false,"sensitive":false},"output-grouping":{"name":"output-grouping","displayName":"Output Grouping","identifiesControllerService":false,"sensitive":false},"schema-name":{"name":"schema-name","displayName":"Schema Name","identifiesControllerService":false,"sensitive":false},"schema-registry":{"name":"schema-registry","displayName":"Schema Registry","identifiesControllerService":true,"sensitive":false},"Time Format":{"name":"Time Format","displayName":"Time Format","identifiesControllerService":false,"sensitive":false},"schema-access-strategy":{"name":"schema-access-strategy","displayName":"Schema Access Strategy","identifiesControllerService":false,"sensitive":false},"schema-protocol-version":{"name":"schema-protocol-version","displayName":"Schema Protocol Version","identifiesControllerService":false,"sensitive":false},"schema-version":{"name":"schema-version","displayName":"Schema Version","identifiesControllerService":false,"sensitive":false},"schema-text":{"name":"schema-text","displayName":"Schema Text","identifiesControllerService":false,"sensitive":false}},"controllerServiceApis":[{"type":"org.apache.nifi.serialization.RecordSetWriterFactory","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"1.16.0.2.2.5.1-1"}}],"scheduledState":"ENABLED","componentType":"CONTROLLER_SERVICE","groupIdentifier":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172"},{"identifier":"c602ce32-0f17-397f-87da-00d6c873454c","instanceIdentifier":"74d630ce-7fbc-3993-a9f8-7f09bc34159b","name":"HortonworksSchemaRegistry","comments":"","type":"org.apache.nifi.schemaregistry.hortonworks.HortonworksSchemaRegistry","bundle":{"group":"org.apache.nifi","artifact":"nifi-hwx-schema-registry-nar","version":"1.16.0.2.2.5.1-1"},"properties":{"cache-expiration":"1 hour","kerberos-credentials-service":null,"kerberos-password":"#{CDP Workload User Password}","ssl-context-service":"b2790196-adee-3d53-ba8d-3a777ccb68be","cache-size":"1000","kerberos-principal":"#{CDP Workload User}","url":"#{Schema Registry Url}"},"propertyDescriptors":{"cache-expiration":{"name":"cache-expiration","displayName":"Cache Expiration","identifiesControllerService":false,"sensitive":false},"kerberos-credentials-service":{"name":"kerberos-credentials-service","displayName":"Kerberos Credentials Service","identifiesControllerService":true,"sensitive":false},"kerberos-password":{"name":"kerberos-password","displayName":"Kerberos Password","identifiesControllerService":false,"sensitive":true},"ssl-context-service":{"name":"ssl-context-service","displayName":"SSL Context Service","identifiesControllerService":true,"sensitive":false},"cache-size":{"name":"cache-size","displayName":"Cache Size","identifiesControllerService":false,"sensitive":false},"kerberos-principal":{"name":"kerberos-principal","displayName":"Kerberos Principal","identifiesControllerService":false,"sensitive":false},"url":{"name":"url","displayName":"Schema Registry URL","identifiesControllerService":false,"sensitive":false}},"controllerServiceApis":[{"type":"org.apache.nifi.schemaregistry.services.SchemaRegistry","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"1.16.0.2.2.5.1-1"}}],"scheduledState":"ENABLED","componentType":"CONTROLLER_SERVICE","groupIdentifier":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172"},{"identifier":"d8dcb534-91fd-36ca-a079-7c99b6a43dab","instanceIdentifier":"9ac22f83-c188-3e81-b9c0-60269f83b4cc","name":"JsonTreeReader","comments":"","type":"org.apache.nifi.json.JsonTreeReader","bundle":{"group":"org.apache.nifi","artifact":"nifi-record-serialization-services-nar","version":"1.16.0.2.2.5.1-1"},"properties":{"Timestamp Format":null,"schema-inference-cache":null,"Date Format":null,"schema-branch":null,"schema-name":"${schema.name}","schema-registry":"c602ce32-0f17-397f-87da-00d6c873454c","Time Format":null,"schema-access-strategy":"schema-name","schema-version":null,"schema-text":"${avro.schema}"},"propertyDescriptors":{"Timestamp Format":{"name":"Timestamp Format","displayName":"Timestamp Format","identifiesControllerService":false,"sensitive":false},"schema-inference-cache":{"name":"schema-inference-cache","displayName":"Schema Inference Cache","identifiesControllerService":true,"sensitive":false},"Date Format":{"name":"Date Format","displayName":"Date Format","identifiesControllerService":false,"sensitive":false},"schema-branch":{"name":"schema-branch","displayName":"Schema Branch","identifiesControllerService":false,"sensitive":false},"schema-name":{"name":"schema-name","displayName":"Schema Name","identifiesControllerService":false,"sensitive":false},"schema-registry":{"name":"schema-registry","displayName":"Schema Registry","identifiesControllerService":true,"sensitive":false},"Time Format":{"name":"Time Format","displayName":"Time Format","identifiesControllerService":false,"sensitive":false},"schema-access-strategy":{"name":"schema-access-strategy","displayName":"Schema Access Strategy","identifiesControllerService":false,"sensitive":false},"schema-version":{"name":"schema-version","displayName":"Schema Version","identifiesControllerService":false,"sensitive":false},"schema-text":{"name":"schema-text","displayName":"Schema Text","identifiesControllerService":false,"sensitive":false}},"controllerServiceApis":[{"type":"org.apache.nifi.serialization.RecordReaderFactory","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-services-api-nar","version":"1.16.0.2.2.5.1-1"}}],"scheduledState":"ENABLED","componentType":"CONTROLLER_SERVICE","groupIdentifier":"9d4aa10d-0a47-300b-a3e7-75a9a3c07172"}],"variables":{},"parameterContextName":"Workloadmanager_pwd","defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE","componentType":"PROCESS_GROUP"},"externalControllerServices":{"b2790196-adee-3d53-ba8d-3a777ccb68be":{"identifier":"b2790196-adee-3d53-ba8d-3a777ccb68be","name":"Default NiFi SSL Context Service"}},"parameterContexts":{"Workloadmanager_pwd":{"name":"Workloadmanager_pwd","parameters":[{"name":"CDP Workload User Password","description":"","sensitive":true},{"name":"CDP Workload User","description":"","sensitive":false,"value":"steven.matison"},{"name":"Schema Registry Url","description":"","sensitive":false,"value":"https://srm-fraud-detection-csm-master0.se-sandb.a465-9q4k.cloudera.site:7790/api/v1"},{"name":"Kafka Brokers","description":"","sensitive":false,"value":"srm-fraud-detection-csm-corebroker0.se-sandb.a465-9q4k.cloudera.site:9093,srm-fraud-detection-csm-corebroker1.se-sandb.a465-9q4k.cloudera.site:9093,srm-fraud-detection-csm-corebroker2.se-sandb.a465-9q4k.cloudera.site:9093"}],"inheritedParameterContexts":[],"componentType":"PARAMETER_CONTEXT"}},"flowEncodingVersion":"1.0"}