[Infoblox BloxOne DDI] Ingest Pipeline Errors on Empty IP Fields, and Blank Messages Ingested

packages/infoblox_bloxone_ddi/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: 1.11.1
+  changes:
+    - description: "Correct conversion of IP addresses on empty arrays, and drop emtpy messages"
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/7943
 - version: 1.11.0
   changes:
     - description: ECS version updated to 8.10.0.

packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/_dev/test/pipeline/test-pipeline-dhcp-lease.log

@@ -1,2 +1,3 @@
 {"address":"81.2.69.192","client_id":"string","ends":"2022-07-14T11:51:15.417Z","fingerprint":"string","fingerprint_processed":"string","ha_group":"string","hardware":"string","host":"string","hostname":"string","iaid":0,"last_updated":"2022-07-14T11:51:15.417Z","options":{"message":"Hello"},"preferred_lifetime":"2022-07-14T11:51:15.417Z","protocol":"ip6","space":"string","starts":"2022-07-14T11:51:15.417Z","state":"string","type":"string"}
 {"address":"81.2.69.192","client_id":"abc3212caabc","ends":"2022-07-14T11:51:15.417Z","fingerprint":"ab3213cbabab/abc23bca","fingerprint_processed":"12abca32bca32abcd","ha_group":"abc321cdcbda321","hardware":"00:00:5E:00:53:00","host":"admin","hostname":"example.com","iaid":0,"last_updated":"2022-07-14T11:51:15.417Z","options":{"message":"Hello"},"preferred_lifetime":"2022-07-14T11:51:15.417Z","protocol":"ip4","space":"string","starts":"2022-07-14T11:51:15.417Z","state":"used","type":"DHCPv4: DHCPv4 lease"}
+{"results":[]}

packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/_dev/test/pipeline/test-pipeline-dhcp-lease.log-expected.json

@@ -136,6 +136,7 @@
                 "preserve_original_event",
                 "preserve_duplicate_custom_fields"
             ]
-        }
+        },
+        null
     ]
 }

packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/elasticsearch/ingest_pipeline/default.yml

@@ -20,6 +20,8 @@ processors:
   - json:
       field: event.original
       target_field: json
+  - drop:
+      if: ctx.json?.results instanceof List && ctx.json.results.length == 0
   - fingerprint:
       fields:
         - json.starts

packages/infoblox_bloxone_ddi/data_stream/dns_config/_dev/test/pipeline/test-pipeline-dns-config.log

@@ -1,2 +1,4 @@
 {"add_edns_option_in_outgoing_query":true,"comment":"string","created_at":"2022-07-15T06:55:25.978Z","custom_root_ns":[{"address":"81.2.69.192","fqdn":"string","protocol_fqdn":"string"}],"custom_root_ns_enabled":true,"disabled":true,"dnssec_enable_validation":true,"dnssec_enabled":true,"dnssec_root_keys":[{"algorithm":2,"protocol_zone":"string","public_key":"string","sep":true,"zone":"string"}],"dnssec_trust_anchors":[{"algorithm":0,"protocol_zone":"string","public_key":"string","sep":true,"zone":"string"}],"dnssec_validate_expiry":true,"ecs_enabled":true,"ecs_forwarding":true,"ecs_prefix_v4":0,"ecs_prefix_v6":0,"ecs_zones":[{"access":"string","fqdn":"string","protocol_fqdn":"string"}],"edns_udp_size":0,"forwarders":[{"address":"81.2.69.192","fqdn":"string","protocol_fqdn":"string"}],"forwarders_only":true,"gss_tsig_enabled":true,"id":"string","inheritance_sources":{"add_edns_option_in_outgoing_query":{"action":"string","display_name":"string","source":"string","value":true},"custom_root_ns_block":{"action":"string","display_name":"string","source":"string","value":{"custom_root_ns":[{"address":"67.43.156.0","fqdn":"string","protocol_fqdn":"string"}],"custom_root_ns_enabled":true}},"dnssec_validation_block":{"action":"string","display_name":"string","source":"string","value":{"dnssec_enable_validation":true,"dnssec_enabled":true,"dnssec_trust_anchors":[{"algorithm":2,"protocol_zone":"string","public_key":"string","sep":false,"zone":"string"}],"dnssec_validate_expiry":true}},"ecs_block":{"action":"string","display_name":"string","source":"string","value":{"ecs_enabled":false,"ecs_forwarding":true,"ecs_prefix_v4":4,"ecs_prefix_v6":10,"ecs_zones":[{"access":"allow","fqdn":"Test Domain","protocol_fqdn":"string"}]}},"edns_udp_size":{"action":"inherit","display_name":"test display","source":"string","value":10},"forwarders_block":{"action":"string","display_name":"string","source":"string","value":{"forwarders":[{"address":"89.160.20.128","fqdn":"string","protocol_fqdn":"string"}],"forwarders_only":true}},"gss_tsig_enabled":{"action":"string","display_name":"string","source":"string","value":true},"lame_ttl":{"action":"string","display_name":"string","source":"string","value":0},"match_recursive_only":{"action":"string","display_name":"string","source":"string","value":false},"max_cache_ttl":{"action":"string","display_name":"string","source":"string","value":0},"max_negative_ttl":{"action":"string","display_name":"string","source":"string","value":12},"max_udp_size":{"action":"string","display_name":"string","source":"string","value":0},"minimal_responses":{"action":"string","display_name":"string","source":"string","value":true},"notify":{"action":"string","display_name":"string","source":"string","value":true},"query_acl":{"action":"deny","display_name":"string","source":"string","value":[{"access":"allow","acl":"string","address":"89.160.20.128","element":"any","tsig_key":{"algorithm":"hmac_sha256","comment":"string","key":"string","name":"string","protocol_name":"string","secret":"string"}}]},"recursion_acl":{"action":"string","display_name":"string","source":"string","value":[{"access":"deny","acl":"string","address":"89.160.20.128","element":"tsig_key","tsig_key":{"algorithm":"hmac_sha384","comment":"string","key":"string","name":"string","protocol_name":"string","secret":"string"}}]},"recursion_enabled":{"action":"string","display_name":"string","source":"string","value":true},"synthesize_address_records_from_https":{"action":"inherit","display_name":"string","source":"string","value":true},"transfer_acl":{"action":"inherit","display_name":"string","source":"string","value":[{"access":"allow","acl":"string","address":"216.160.83.56","element":"string","tsig_key":{"algorithm":"hmac_sha224","comment":"string","key":"string","name":"string","protocol_name":"string","secret":"string"}}]},"update_acl":{"action":"string","display_name":"string","source":"string","value":[{"access":"allow","acl":"string","address":"216.160.83.56","element":"acl","tsig_key":{"algorithm":"hmac_sha384","comment":"string","key":"string","name":"string","protocol_name":"string","secret":"string"}}]},"use_forwarders_for_subzones":{"action":"override","display_name":"string","source":"string","value":false},"zone_authority":{"default_ttl":{"action":"string","display_name":"string","source":"string","value":0},"expire":{"action":"string","display_name":"string","source":"string","value":0},"mname_block":{"action":"string","display_name":"string","source":"string","value":{"mname":"string","protocol_mname":"Test server","use_default_mname":true}},"negative_ttl":{"action":"string","display_name":"string","source":"string","value":0},"protocol_rname":{"action":"string","display_name":"string","source":"string","value":"string"},"refresh":{"action":"string","display_name":"string","source":"string","value":0},"retry":{"action":"string","display_name":"string","source":"string","value":0},"rname":{"action":"string","display_name":"string","source":"string","value":"string"}}},"ip_spaces":["string"],"lame_ttl":0,"match_clients_acl":[{"access":"deny","acl":"string","address":"81.2.69.192","element":"any","tsig_key":{"algorithm":"hmac_sha512","comment":"string","key":"string","name":"string","protocol_name":"string","secret":"string"}}],"match_destinations_acl":[{"access":"allow","acl":"test acl","address":"81.2.69.192","element":"string","tsig_key":{"algorithm":"hmac_sha384","comment":"string","key":"string","name":"string","protocol_name":"string","secret":"string"}}],"match_recursive_only":true,"max_cache_ttl":0,"max_negative_ttl":0,"max_udp_size":0,"minimal_responses":true,"name":"string","notify":true,"query_acl":[{"access":"string","acl":"string","address":"81.2.69.192","element":"acl","tsig_key":{"algorithm":"hmac_sha224","comment":"testing comment","key":"string","name":"string","protocol_name":"string","secret":"string"}}],"recursion_acl":[{"access":"allow","acl":"ACL","address":"81.2.69.192","element":"any","tsig_key":{"algorithm":"hmac_sha1","comment":"testing_comment","key":"test key","name":"string","protocol_name":"string","secret":"string"}}],"recursion_enabled":true,"synthesize_address_records_from_https":false,"tags":{},"transfer_acl":[{"access":"allow","acl":"value","address":"216.160.83.56","element":"any","tsig_key":{"algorithm":"hmac_sha224","comment":"string","key":"test","name":"string","protocol_name":"string","secret":"string"}}],"update_acl":[{"access":"allow","acl":"name","address":"216.160.83.56","element":"acl","tsig_key":{"algorithm":"hmac_sha1","comment":"string","key":"Test","name":"string","protocol_name":"string","secret":"string"}}],"updated_at":"2022-07-15T06:55:25.978Z","use_forwarders_for_subzones":true,"zone_authority":{"default_ttl":0,"expire":0,"mname":"string","negative_ttl":0,"protocol_mname":"string","protocol_rname":"string","refresh":0,"retry":0,"rname":"string","use_default_mname":true}}
 {"add_edns_option_in_outgoing_query":false,"comment":"","created_at":"2022-08-16T09:24:24.231424Z","custom_root_ns":[],"custom_root_ns_enabled":false,"disabled":false,"dnssec_enable_validation":true,"dnssec_enabled":true,"dnssec_root_keys":[{"algorithm":2,"protocol_zone":".","public_key":"abc12dc34/12abcd3242cdaaacd+/abd12cdacd4221cdacbdbdbdbd/abcAC23CBD22323abcddcba=/abcddcbdcda13411bdbddb=","sep":true,"zone":"."}],"dnssec_trust_anchors":[],"dnssec_validate_expiry":true,"ecs_enabled":false,"ecs_forwarding":false,"ecs_prefix_v4":24,"ecs_prefix_v6":56,"ecs_zones":[],"edns_udp_size":1232,"forwarders":[],"forwarders_only":false,"gss_tsig_enabled":false,"id":"dns/view/abcd-12acbd-1ab2-12abcd-1abcd33","inheritance_sources":null,"ip_spaces":["ipam/ip_space/1abcd323-12abcd-12abcd-12ab-123badcd"],"lame_ttl":600,"match_clients_acl":[{"access":"allow","acl":null,"address":"81.2.69.192","element":"any","tsig_key":null}],"match_destinations_acl":[{"access":"allow","acl":null,"address":"81.2.69.192","element":"any","tsig_key":null}],"match_recursive_only":false,"max_cache_ttl":604800,"max_negative_ttl":10800,"max_udp_size":1232,"minimal_responses":false,"name":"test name","notify":false,"query_acl":[],"recursion_acl":[],"recursion_enabled":true,"synthesize_address_records_from_https":false,"tags":null,"transfer_acl":[],"update_acl":[],"updated_at":"2022-08-16T09:24:24.231424Z","use_forwarders_for_subzones":true,"zone_authority":{"default_ttl":28800,"expire":2419200,"mname":"ns.b1ddi","negative_ttl":900,"protocol_mname":"ns.b1ddi","protocol_rname":"hostmaster","refresh":10800,"retry":3600,"rname":"hostmaster","use_default_mname":true}}
+{"add_edns_option_in_outgoing_query":false,"comment":"","created_at":"2023-09-14T18:54:45.215640Z","custom_root_ns":[],"custom_root_ns_enabled":false,"disabled":false,"dnssec_enable_validation":true,"dnssec_enabled":true,"dnssec_root_keys":[{"algorithm":8,"protocol_zone":".","public_key":"Loremips/Loremipsumdolorsitametconsecteturadipiscingelit+/eddoeiusmodtemporincididuntutlaboreetdo+loremagnaaliquaUtenimadminimveniamquisnostrudexercitationullamcolaborisnisiutaliquipe/xeacommodoconsequ/tDuisauteiruredolorinreprehenderitinvoluptatevelitessecillumdolo/reeufugiatnullapariaturExcepteursintoccaecatcupidatatnonproidentsuntinculpaquiof=","sep":true,"zone":"."}],"dnssec_trust_anchors":[],"dnssec_validate_expiry":true,"ecs_enabled":false,"ecs_forwarding":false,"ecs_prefix_v4":24,"ecs_prefix_v6":56,"ecs_zones":[],"edns_udp_size":1232,"filter_aaaa_acl":[],"filter_aaaa_on_v4":"no","forwarders":[],"forwarders_only":false,"gss_tsig_enabled":false,"id":"dns/view/01234567-89ab-cdef-fedc-ba9876543210","inheritance_sources":null,"ip_spaces":[],"lame_ttl":600,"match_clients_acl":[{"access":"allow","acl":null,"address":"","element":"any","tsig_key":null}],"match_destinations_acl":[{"access":"allow","acl":null,"address":"","element":"any","tsig_key":null}],"match_recursive_only":false,"max_cache_ttl":604800,"max_negative_ttl":10800,"max_udp_size":1232,"minimal_responses":false,"name":"default-Contoso","notify":false,"query_acl":[],"recursion_acl":[],"recursion_enabled":false,"sort_list":[],"synthesize_address_records_from_https":false,"tags":{"nios/grid_name":"Contoso","nios/imported":"true"},"transfer_acl":[],"update_acl":[],"updated_at":"2023-09-14T18:54:45.215640Z","use_forwarders_for_subzones":true,"use_root_forwarders_for_local_resolution_with_b1td":false,"zone_authority":{"default_ttl":28800,"expire":2419200,"mname":"ns.b1ddi","negative_ttl":900,"protocol_mname":"ns.b1ddi","protocol_rname":"hostmaster","refresh":10800,"retry":3600,"rname":"hostmaster","use_default_mname":true}}
+{"results":[]}

packages/infoblox_bloxone_ddi/data_stream/dns_config/_dev/test/pipeline/test-pipeline-dns-config.log-expected.json

@@ -759,6 +759,120 @@
                 "preserve_original_event",
                 "preserve_duplicate_custom_fields"
             ]
-        }
+        },
+        {
+            "@timestamp": "2023-09-14T18:54:45.215Z",
+            "dns": {
+                "answers": {
+                    "ttl": 600
+                }
+            },
+            "ecs": {
+                "version": "8.10.0"
+            },
+            "event": {
+                "category": [
+                    "network"
+                ],
+                "created": "2023-09-14T18:54:45.215Z",
+                "id": "dns/view/01234567-89ab-cdef-fedc-ba9876543210",
+                "kind": "event",
+                "original": "{\"add_edns_option_in_outgoing_query\":false,\"comment\":\"\",\"created_at\":\"2023-09-14T18:54:45.215640Z\",\"custom_root_ns\":[],\"custom_root_ns_enabled\":false,\"disabled\":false,\"dnssec_enable_validation\":true,\"dnssec_enabled\":true,\"dnssec_root_keys\":[{\"algorithm\":8,\"protocol_zone\":\".\",\"public_key\":\"Loremips/Loremipsumdolorsitametconsecteturadipiscingelit+/eddoeiusmodtemporincididuntutlaboreetdo+loremagnaaliquaUtenimadminimveniamquisnostrudexercitationullamcolaborisnisiutaliquipe/xeacommodoconsequ/tDuisauteiruredolorinreprehenderitinvoluptatevelitessecillumdolo/reeufugiatnullapariaturExcepteursintoccaecatcupidatatnonproidentsuntinculpaquiof=\",\"sep\":true,\"zone\":\".\"}],\"dnssec_trust_anchors\":[],\"dnssec_validate_expiry\":true,\"ecs_enabled\":false,\"ecs_forwarding\":false,\"ecs_prefix_v4\":24,\"ecs_prefix_v6\":56,\"ecs_zones\":[],\"edns_udp_size\":1232,\"filter_aaaa_acl\":[],\"filter_aaaa_on_v4\":\"no\",\"forwarders\":[],\"forwarders_only\":false,\"gss_tsig_enabled\":false,\"id\":\"dns/view/01234567-89ab-cdef-fedc-ba9876543210\",\"inheritance_sources\":null,\"ip_spaces\":[],\"lame_ttl\":600,\"match_clients_acl\":[{\"access\":\"allow\",\"acl\":null,\"address\":\"\",\"element\":\"any\",\"tsig_key\":null}],\"match_destinations_acl\":[{\"access\":\"allow\",\"acl\":null,\"address\":\"\",\"element\":\"any\",\"tsig_key\":null}],\"match_recursive_only\":false,\"max_cache_ttl\":604800,\"max_negative_ttl\":10800,\"max_udp_size\":1232,\"minimal_responses\":false,\"name\":\"default-Contoso\",\"notify\":false,\"query_acl\":[],\"recursion_acl\":[],\"recursion_enabled\":false,\"sort_list\":[],\"synthesize_address_records_from_https\":false,\"tags\":{\"nios/grid_name\":\"Contoso\",\"nios/imported\":\"true\"},\"transfer_acl\":[],\"update_acl\":[],\"updated_at\":\"2023-09-14T18:54:45.215640Z\",\"use_forwarders_for_subzones\":true,\"use_root_forwarders_for_local_resolution_with_b1td\":false,\"zone_authority\":{\"default_ttl\":28800,\"expire\":2419200,\"mname\":\"ns.b1ddi\",\"negative_ttl\":900,\"protocol_mname\":\"ns.b1ddi\",\"protocol_rname\":\"hostmaster\",\"refresh\":10800,\"retry\":3600,\"rname\":\"hostmaster\",\"use_default_mname\":true}}",
+                "type": [
+                    "protocol"
+                ]
+            },
+            "infoblox_bloxone_ddi": {
+                "dns_config": {
+                    "add_edns": {
+                        "option_in": {
+                            "outgoing_query": false
+                        }
+                    },
+                    "created_at": "2023-09-14T18:54:45.215Z",
+                    "custom_root_ns_enabled": false,
+                    "disabled": false,
+                    "dnssec": {
+                        "enable_validation": true,
+                        "enabled": true,
+                        "root_keys": [
+                            {
+                                "algorithm": 8,
+                                "protocol": {
+                                    "zone": "."
+                                },
+                                "public": "Loremips/Loremipsumdolorsitametconsecteturadipiscingelit+/eddoeiusmodtemporincididuntutlaboreetdo+loremagnaaliquaUtenimadminimveniamquisnostrudexercitationullamcolaborisnisiutaliquipe/xeacommodoconsequ/tDuisauteiruredolorinreprehenderitinvoluptatevelitessecillumdolo/reeufugiatnullapariaturExcepteursintoccaecatcupidatatnonproidentsuntinculpaquiof=",
+                                "sep": true,
+                                "zone": "."
+                            }
+                        ],
+                        "validate_expiry": true
+                    },
+                    "ecs": {
+                        "enabled": false,
+                        "forwarding": false,
+                        "prefix_v4": 24,
+                        "prefix_v6": 56
+                    },
+                    "edns": {
+                        "udp": {
+                            "size": 1232
+                        }
+                    },
+                    "forwarders_only": false,
+                    "gss_tsig_enabled": false,
+                    "id": "dns/view/01234567-89ab-cdef-fedc-ba9876543210",
+                    "lame_ttl": 600,
+                    "match_clients_acl": [
+                        {
+                            "access": "allow",
+                            "element": "any"
+                        }
+                    ],
+                    "match_destinations_acl": [
+                        {
+                            "access": "allow",
+                            "element": "any"
+                        }
+                    ],
+                    "match_recursive_only": false,
+                    "max_cache_ttl": 604800,
+                    "max_negative_ttl": 10800,
+                    "max_udp_size": 1232,
+                    "minimal_responses": false,
+                    "name": "default-Contoso",
+                    "notify": false,
+                    "recursion_enabled": false,
+                    "synthesize": {
+                        "address_records_from_https": false
+                    },
+                    "tags": {
+                        "nios/grid_name": "Contoso",
+                        "nios/imported": "true"
+                    },
+                    "updated_at": "2023-09-14T18:54:45.215Z",
+                    "use_forwarders_for_subzones": true,
+                    "zone_authority": {
+                        "default_ttl": 28800,
+                        "expire": 2419200,
+                        "mname": "ns.b1ddi",
+                        "negative_ttl": 900,
+                        "protocol": {
+                            "mname": "ns.b1ddi",
+                            "rname": "hostmaster"
+                        },
+                        "refresh": 10800,
+                        "retry": 3600,
+                        "rname": "hostmaster",
+                        "use_default_mname": true
+                    }
+                }
+            },
+            "tags": [
+                "preserve_original_event",
+                "preserve_duplicate_custom_fields"
+            ]
+        },
+        null
     ]
 }