Skip to content

Commit

Permalink
infoblox_bloxone_ddi: fix handling of options fields
Browse files Browse the repository at this point in the history
  • Loading branch information
@efd6
efd6 committed Oct 4, 2023
1 parent 70a8199 commit 63c4bc6
Show file tree
Hide file tree
Showing 11 changed files with 927 additions and 4 deletions.
673 changes: 670 additions & 3 deletions packages/infoblox_bloxone_ddi/_dev/deploy/docker/files/config.yml
View file

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions packages/infoblox_bloxone_ddi/changelog.yml
View file
@@ -1,4 +1,9 @@
# newer versions go on top
- version: 1.11.2
changes:
- description: Fix handling of `infoblox_bloxone_ddi.dhcp_lease.options` field.
type: bugfix
link: https://github.com/elastic/integrations/pull/1
- version: 1.11.1
changes:
- description: "Correct conversion of IP addresses on empty arrays, and drop emtpy messages"
Expand Down
1 change: 1 addition & 0 deletions ...foblox_bloxone_ddi/data_stream/dhcp_lease/_dev/test/pipeline/test-pipeline-dhcp-lease.log
View file
@@ -1,3 +1,4 @@
{"address":"81.2.69.192","client_id":"string","ends":"2022-07-14T11:51:15.417Z","fingerprint":"string","fingerprint_processed":"string","ha_group":"string","hardware":"string","host":"string","hostname":"string","iaid":0,"last_updated":"2022-07-14T11:51:15.417Z","options":{"message":"Hello"},"preferred_lifetime":"2022-07-14T11:51:15.417Z","protocol":"ip6","space":"string","starts":"2022-07-14T11:51:15.417Z","state":"string","type":"string"}
{"address":"81.2.69.192","client_id":"abc3212caabc","ends":"2022-07-14T11:51:15.417Z","fingerprint":"ab3213cbabab/abc23bca","fingerprint_processed":"12abca32bca32abcd","ha_group":"abc321cdcbda321","hardware":"00:00:5E:00:53:00","host":"admin","hostname":"example.com","iaid":0,"last_updated":"2022-07-14T11:51:15.417Z","options":{"message":"Hello"},"preferred_lifetime":"2022-07-14T11:51:15.417Z","protocol":"ip4","space":"string","starts":"2022-07-14T11:51:15.417Z","state":"used","type":"DHCPv4: DHCPv4 lease"}
{"address":"10.10.10.10","client_id":"","ends":"2023-09-22T00:27:00Z","fingerprint":"System Name","fingerprint_processed":"processed","ha_group":"dhcp/ha_group/01234567-89ab-cdef-fedc-ba9876543210","hardware":"00:11:22:33:44:55","host":"dhcp/host/123456","hostname":"system_name.contoso.com","iaid":0,"last_updated":"2023-09-21T20:27:00.774Z","options":"{\"Options\":[{\"Code\":\"51\",\"Value\":\"AAA4QA==\"},{\"Code\":\"53\",\"Value\":\"Aw==\"},{\"Code\":\"55\",\"Value\":\"AQIDBAYPKjYHoEIr\"},{\"Code\":\"6\",\"Value\":\"CmQGYwozAGM=\"},{\"Code\":\"60\",\"Value\":\"UG9seWNvbS1TU0lQNzAwMA==\"},{\"Code\":\"1\",\"Value\":\"///+AA==\"},{\"Code\":\"125\",\"Value\":\"LoremipsumdolorsitametconsecteturadipiscingelitseddoeiusmodtemporincididuntutlaboreetdoloremagnaaliquaUtenimadminimveniamquisnostrudexercitationullamcolaborisnisiut\"},{\"Code\":\"12\",\"Value\":\"Loremipsumdolorsitametconse=\"},{\"Code\":\"3\",\"Value\":\"CnVgAQ==\"}]}","preferred_lifetime":"2023-09-21T20:27:00Z","protocol":"","space":"ipam/ip_space/01234567-89ab-cdef-fedc-ba9876543210","starts":"2023-09-21T20:27:00Z","state":"used","type":"DHCPv4"}
{"results":[]}
96 changes: 96 additions & 0 deletions ..._ddi/data_stream/dhcp_lease/_dev/test/pipeline/test-pipeline-dhcp-lease.log-expected.json
View file
Expand Up @@ -137,6 +137,102 @@
"preserve_duplicate_custom_fields"
]
},
{
"@timestamp": "2023-09-21T20:27:00.774Z",
"ecs": {
"version": "8.10.0"
},
"event": {
"category": [
"network"
],
"end": "2023-09-22T00:27:00.000Z",
"kind": "event",
"original": "{\"address\":\"10.10.10.10\",\"client_id\":\"\",\"ends\":\"2023-09-22T00:27:00Z\",\"fingerprint\":\"System Name\",\"fingerprint_processed\":\"processed\",\"ha_group\":\"dhcp/ha_group/01234567-89ab-cdef-fedc-ba9876543210\",\"hardware\":\"00:11:22:33:44:55\",\"host\":\"dhcp/host/123456\",\"hostname\":\"system_name.contoso.com\",\"iaid\":0,\"last_updated\":\"2023-09-21T20:27:00.774Z\",\"options\":\"{\\\"Options\\\":[{\\\"Code\\\":\\\"51\\\",\\\"Value\\\":\\\"AAA4QA==\\\"},{\\\"Code\\\":\\\"53\\\",\\\"Value\\\":\\\"Aw==\\\"},{\\\"Code\\\":\\\"55\\\",\\\"Value\\\":\\\"AQIDBAYPKjYHoEIr\\\"},{\\\"Code\\\":\\\"6\\\",\\\"Value\\\":\\\"CmQGYwozAGM=\\\"},{\\\"Code\\\":\\\"60\\\",\\\"Value\\\":\\\"UG9seWNvbS1TU0lQNzAwMA==\\\"},{\\\"Code\\\":\\\"1\\\",\\\"Value\\\":\\\"///+AA==\\\"},{\\\"Code\\\":\\\"125\\\",\\\"Value\\\":\\\"LoremipsumdolorsitametconsecteturadipiscingelitseddoeiusmodtemporincididuntutlaboreetdoloremagnaaliquaUtenimadminimveniamquisnostrudexercitationullamcolaborisnisiut\\\"},{\\\"Code\\\":\\\"12\\\",\\\"Value\\\":\\\"Loremipsumdolorsitametconse=\\\"},{\\\"Code\\\":\\\"3\\\",\\\"Value\\\":\\\"CnVgAQ==\\\"}]}\",\"preferred_lifetime\":\"2023-09-21T20:27:00Z\",\"protocol\":\"\",\"space\":\"ipam/ip_space/01234567-89ab-cdef-fedc-ba9876543210\",\"starts\":\"2023-09-21T20:27:00Z\",\"state\":\"used\",\"type\":\"DHCPv4\"}",
"start": "2023-09-21T20:27:00.000Z",
"type": [
"protocol"
]
},
"host": {
"hostname": "system_name.contoso.com",
"name": "dhcp/host/123456"
},
"infoblox_bloxone_ddi": {
"dhcp_lease": {
"address": "10.10.10.10",
"ends": "2023-09-22T00:27:00.000Z",
"fingerprint": {
"processed": "processed",
"value": "System Name"
},
"ha_group": "dhcp/ha_group/01234567-89ab-cdef-fedc-ba9876543210",
"hardware": "00-11-22-33-44-55",
"host": "dhcp/host/123456",
"hostname": "system_name.contoso.com",
"iaid": 0,
"last_updated": "2023-09-21T20:27:00.774Z",
"options": {
"Options": [
{
"Code": "51",
"Value": "AAA4QA=="
},
{
"Code": "53",
"Value": "Aw=="
},
{
"Code": "55",
"Value": "AQIDBAYPKjYHoEIr"
},
{
"Code": "6",
"Value": "CmQGYwozAGM="
},
{
"Code": "60",
"Value": "UG9seWNvbS1TU0lQNzAwMA=="
},
{
"Code": "1",
"Value": "///+AA=="
},
{
"Code": "125",
"Value": "LoremipsumdolorsitametconsecteturadipiscingelitseddoeiusmodtemporincididuntutlaboreetdoloremagnaaliquaUtenimadminimveniamquisnostrudexercitationullamcolaborisnisiut"
},
{
"Code": "12",
"Value": "Loremipsumdolorsitametconse="
},
{
"Code": "3",
"Value": "CnVgAQ=="
}
]
},
"preferred_lifetime": "2023-09-21T20:27:00.000Z",
"space": "ipam/ip_space/01234567-89ab-cdef-fedc-ba9876543210",
"starts": "2023-09-21T20:27:00.000Z",
"state": "used",
"type": "DHCPv4"
}
},
"related": {
"hosts": [
"dhcp/host/123456",
"system_name.contoso.com"
],
"ip": [
"10.10.10.10"
]
},
"tags": [
"preserve_original_event",
"preserve_duplicate_custom_fields"
]
},
null
]
}
2 changes: 2 additions & 0 deletions ...ages/infoblox_bloxone_ddi/data_stream/dhcp_lease/_dev/test/system/test-default-config.yml
View file
Expand Up @@ -8,3 +8,5 @@ data_stream:
vars:
preserve_original_event: true
preserve_duplicate_custom_fields: true
assert:
hit_count: 2
11 changes: 11 additions & 0 deletions ...ges/infoblox_bloxone_ddi/data_stream/dhcp_lease/elasticsearch/ingest_pipeline/default.yml
View file
Expand Up @@ -203,6 +203,17 @@ processors:
field: json.type
target_field: infoblox_bloxone_ddi.dhcp_lease.type
ignore_missing: true
- json:
field: infoblox_bloxone_ddi.dhcp_lease.options
target_field: infoblox_bloxone_ddi.dhcp_lease.options
if: ctx.infoblox_bloxone_ddi?.dhcp_lease?.options instanceof String
on_failure:
- remove:
field: infoblox_bloxone_ddi.dhcp_lease.options
ignore_failure: true
- append:
field: error.message
value: '{{{_ingest.on_failure_message}}}'
- remove:
field: json
ignore_missing: true
Expand Down
1 change: 1 addition & 0 deletions ...s/infoblox_bloxone_ddi/data_stream/dns_data/_dev/test/pipeline/test-pipeline-dns-data.log
View file
@@ -1,3 +1,4 @@
{"absolute_name_spec":"string","absolute_zone_name":"string","comment":"string","created_at":"2022-07-20T09:59:59.184Z","delegation":"string","disabled":true,"dns_absolute_name_spec":"string","dns_absolute_zone_name":"string","dns_name_in_zone":"string","dns_rdata":"string","id":"string","inheritance_sources":{"ttl":{"action":"string","display_name":"string","source":"string","value":0}},"name_in_zone":"string","options":{},"rdata":{"address":"81.2.69.192","flags":"falg_value","tag":"issue","value":"value","cname":"canonical name","target":".","dhcid":"122zbczba12","exchange":"domain name","preference":12345363467,"order":123124,"regexp":"^sqq:","replacement":"","services":"","dname":"domain name","expire":23131,"mname":"test mname","negative_ttl":213342,"refresh":10800,"retry":3600,"rname":"test","serial":12314114,"port":80,"priority":44,"weight":0,"text":"text field","type":"32BIT","length_kind":8},"source":["string"],"tags":{},"ttl":0,"type":"string","updated_at":"2022-07-20T09:59:59.184Z","view":"string","view_name":"string","zone":"string"}
{"absolute_name_spec":"test_spec","absolute_zone_name":"www.example.com","comment":"string","created_at":"2022-07-14T11:50:28.838Z","delegation":"abc123abcd323","disabled":true,"dns_absolute_name_spec":"Test name","dns_absolute_zone_name":"test Zone","dns_name_in_zone":"Test zone","dns_rdata":"DNS rdata","id":"12abcddcba32ab","inheritance_sources":{"ttl":{"action":"inherit","display_name":"test Display","source":"12abc321ab","value":0}},"name_in_zone":"test zone","options":{"address":"81.2.69.144"},"rdata":{"address":"81.2.69.142","value":"wefewf"},"source":["STATIC"],"tags":{},"ttl":0,"type":"AAAA","updated_at":"2022-07-14T11:50:28.838Z","view":"12abcd32bcd12","view_name":"string","zone":"123bcdacd32"}
{"absolute_name_spec":"DNS Data Absolute Name","absolute_zone_name":"DNS Data Absolute Zone Name","comment":"DNS Data Comment","created_at":"2022-07-21T09:59:59.184Z","delegation":"DNS Data Delegation","disabled":true,"dns_absolute_name_spec":"DNS Absolute Name","dns_absolute_zone_name":"DNS Absolute Zone Name","dns_name_in_zone":"DNS Name in Zone","dns_rdata":"DNS RData","id":"ghr123ghf","inheritance_sources":{"ttl":{"action":"DNS Data Action","display_name":"DNS Display Name","source":"DNS Data Source","value":10}},"name_in_zone":"DNS Data Name in zone","options":"{\"create_ptr\":false,\"check_rmz\":true,\"address\":\"67.43.156.0\"}","rdata":{"address":"81.2.69.192","flags":"DNS Data Flags","tag":"issue","value":"DNS Data Value","cname":"DNS Data Canonical Name","target":"DNS Data Target","dhcid":"122zbczba12","exchange":"DNS Data Exchange","preference":12345363467,"order":123124,"regexp":"none","replacement":"DNS Data Replacement","services":"DNS Data Test Services","dname":"DNS Data dname","expire":23131,"mname":"DNS Data mname","negative_ttl":213342,"refresh":10800,"retry":3600,"rname":"DNS Data rname","serial":12314114,"port":80,"priority":44,"weight":0,"text":"DNS Data text field","type":"32BIT","length_kind":8},"source":["STATIC"],"tags":{"message":"Hello"},"ttl":0,"type":"DNS Data Type","updated_at":"2022-07-20T09:59:59.184Z","view":"DNS Data View","view_name":"DNS Data View Name","zone":"DNS Data Zone"}
{"results":[]}
119 changes: 119 additions & 0 deletions ...xone_ddi/data_stream/dns_data/_dev/test/pipeline/test-pipeline-dns-data.log-expected.json
View file
Expand Up @@ -197,6 +197,125 @@
"preserve_duplicate_custom_fields"
]
},
{
"@timestamp": "2022-07-20T09:59:59.184Z",
"dns": {
"answers": {
"ttl": 0
}
},
"ecs": {
"version": "8.10.0"
},
"event": {
"category": [
"network"
],
"created": "2022-07-21T09:59:59.184Z",
"id": "ghr123ghf",
"kind": "event",
"original": "{\"absolute_name_spec\":\"DNS Data Absolute Name\",\"absolute_zone_name\":\"DNS Data Absolute Zone Name\",\"comment\":\"DNS Data Comment\",\"created_at\":\"2022-07-21T09:59:59.184Z\",\"delegation\":\"DNS Data Delegation\",\"disabled\":true,\"dns_absolute_name_spec\":\"DNS Absolute Name\",\"dns_absolute_zone_name\":\"DNS Absolute Zone Name\",\"dns_name_in_zone\":\"DNS Name in Zone\",\"dns_rdata\":\"DNS RData\",\"id\":\"ghr123ghf\",\"inheritance_sources\":{\"ttl\":{\"action\":\"DNS Data Action\",\"display_name\":\"DNS Display Name\",\"source\":\"DNS Data Source\",\"value\":10}},\"name_in_zone\":\"DNS Data Name in zone\",\"options\":\"{\\\"create_ptr\\\":false,\\\"check_rmz\\\":true,\\\"address\\\":\\\"67.43.156.0\\\"}\",\"rdata\":{\"address\":\"81.2.69.192\",\"flags\":\"DNS Data Flags\",\"tag\":\"issue\",\"value\":\"DNS Data Value\",\"cname\":\"DNS Data Canonical Name\",\"target\":\"DNS Data Target\",\"dhcid\":\"122zbczba12\",\"exchange\":\"DNS Data Exchange\",\"preference\":12345363467,\"order\":123124,\"regexp\":\"none\",\"replacement\":\"DNS Data Replacement\",\"services\":\"DNS Data Test Services\",\"dname\":\"DNS Data dname\",\"expire\":23131,\"mname\":\"DNS Data mname\",\"negative_ttl\":213342,\"refresh\":10800,\"retry\":3600,\"rname\":\"DNS Data rname\",\"serial\":12314114,\"port\":80,\"priority\":44,\"weight\":0,\"text\":\"DNS Data text field\",\"type\":\"32BIT\",\"length_kind\":8},\"source\":[\"STATIC\"],\"tags\":{\"message\":\"Hello\"},\"ttl\":0,\"type\":\"DNS Data Type\",\"updated_at\":\"2022-07-20T09:59:59.184Z\",\"view\":\"DNS Data View\",\"view_name\":\"DNS Data View Name\",\"zone\":\"DNS Data Zone\"}",
"type": [
"protocol"
]
},
"infoblox_bloxone_ddi": {
"dns_data": {
"absolute": {
"name": {
"spec": "DNS Absolute Name"
},
"zone": {
"name": "DNS Absolute Zone Name"
}
},
"absolute_name": {
"spec": "DNS Data Absolute Name"
},
"absolute_zone": {
"name": "DNS Data Absolute Zone Name"
},
"comment": "DNS Data Comment",
"created_at": "2022-07-21T09:59:59.184Z",
"delegation": "DNS Data Delegation",
"disabled": true,
"id": "ghr123ghf",
"inheritance": {
"sources": {
"ttl": {
"action": "DNS Data Action",
"display": {
"name": "DNS Display Name"
},
"source": "DNS Data Source",
"value": 10
}
}
},
"name_in": {
"zone": "DNS Name in Zone"
},
"name_in_zone": "DNS Data Name in zone",
"options": {
"address": "67.43.156.0",
"check_rmz": true,
"create_ptr": false
},
"rdata": {
"address": "81.2.69.192",
"cname": "DNS Data Canonical Name",
"dhcid": "122zbczba12",
"dname": "DNS Data dname",
"exchange": "DNS Data Exchange",
"expire": 23131,
"flags": "DNS Data Flags",
"length_kind": 8,
"mname": "DNS Data mname",
"negative_ttl": 213342,
"order": 123124,
"port": 80,
"preference": 12345363467,
"priority": 44,
"refresh": 10800,
"regexp": "none",
"replacement": "DNS Data Replacement",
"retry": 3600,
"rname": "DNS Data rname",
"serial": 12314114,
"services": "DNS Data Test Services",
"tag": "issue",
"target": "DNS Data Target",
"text": "DNS Data text field",
"type": "32BIT",
"value": "DNS Data Value",
"weight": 0
},
"rdata_value": "DNS RData",
"source": [
"STATIC"
],
"tags": {
"message": "Hello"
},
"ttl": 0,
"type": "DNS Data Type",
"updated_at": "2022-07-20T09:59:59.184Z",
"view": "DNS Data View",
"view_name": "DNS Data View Name",
"zone": "DNS Data Zone"
}
},
"related": {
"ip": [
"67.43.156.0",
"81.2.69.192"
]
},
"tags": [
"preserve_original_event",
"preserve_duplicate_custom_fields"
]
},
null
]
}
2 changes: 2 additions & 0 deletions packages/infoblox_bloxone_ddi/data_stream/dns_data/_dev/test/system/test-default-config.yml
View file
Expand Up @@ -8,3 +8,5 @@ data_stream:
vars:
preserve_original_event: true
preserve_duplicate_custom_fields: true
assert:
hit_count: 2
19 changes: 19 additions & 0 deletions packages/infoblox_bloxone_ddi/data_stream/dns_data/elasticsearch/ingest_pipeline/default.yml
View file
Expand Up @@ -119,6 +119,14 @@ processors:
field: json.name_in_zone
target_field: infoblox_bloxone_ddi.dns_data.name_in_zone
ignore_missing: true
- json:
field: json.options
target_field: json.options
if: ctx.json?.options instanceof String
on_failure:
- append:
field: error.message
value: '{{{_ingest.on_failure_message}}}'
- convert:
field: json.options.create_ptr
target_field: infoblox_bloxone_ddi.dns_data.options.create_ptr
Expand Down Expand Up @@ -395,6 +403,17 @@ processors:
field: json.zone
target_field: infoblox_bloxone_ddi.dns_data.zone
ignore_missing: true
- json:
field: infoblox_bloxone_ddi.dns_data.options
target_field: infoblox_bloxone_ddi.dns_data.options
if: ctx.infoblox_bloxone_ddi?.dns_data?.options instanceof String
on_failure:
- remove:
field: infoblox_bloxone_ddi.dns_data.options
ignore_failure: true
- append:
field: error.message
value: '{{{_ingest.on_failure_message}}}'
- remove:
field: json
ignore_missing: true
Expand Down
2 changes: 1 addition & 1 deletion packages/infoblox_bloxone_ddi/manifest.yml
View file
@@ -1,7 +1,7 @@
format_version: "3.0.0"
name: infoblox_bloxone_ddi
title: Infoblox BloxOne DDI
version: "1.11.1"
version: "1.11.2"
description: Collect logs from Infoblox BloxOne DDI with Elastic Agent.
type: integration
categories:
Expand Down

0 comments on commit 63c4bc6

Please sign in to comment.