Skip to content
This repository has been archived by the owner on Dec 17, 2021. It is now read-only.

Commit

Permalink
feat: suffix parsing (#187)
Browse files Browse the repository at this point in the history 
* feat: suffix parsing

* fix: build fix

* fix: build fix

* fix: PR comments

* fix: fix
  • Loading branch information
@weliasz
weliasz committed Nov 4, 2021
1 parent 23f86d5 commit 86611d3
Show file tree
Hide file tree
Showing 4 changed files with 16 additions and 117 deletions.
54 changes: 9 additions & 45 deletions splunk_connect_for_snmp_poller/manager/hec_sender.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
# limitations under the License.
# ########################################################################
import json
import re
import time

import requests
Expand All @@ -27,12 +26,6 @@
)
from splunk_connect_for_snmp_poller.manager.data.inventory_record import InventoryRecord
from splunk_connect_for_snmp_poller.manager.static.mib_enricher import MibEnricher
from splunk_connect_for_snmp_poller.manager.variables import (
enricher_additional_varbinds,
enricher_name,
enricher_oid_family,
)
from splunk_connect_for_snmp_poller.utilities import multi_key_lookup

logger = get_logger(__name__)

Expand Down Expand Up @@ -77,7 +70,6 @@ def post_data_to_splunk_hec(
index,
ir: InventoryRecord,
additional_metric_fields,
server_config,
one_time_flag=False,
mib_enricher=None,
is_error=False,
Expand All @@ -93,7 +85,6 @@ def post_data_to_splunk_hec(
index["metric_index"],
ir,
additional_metric_fields,
server_config,
one_time_flag=one_time_flag,
mib_enricher=mib_enricher,
)
Expand Down Expand Up @@ -184,13 +175,13 @@ def build_metric_data(
index,
ir: InventoryRecord,
additional_metric_fields,
server_config,
one_time_flag=False,
mib_enricher=None,
):
json_val = json.loads(variables_binds)
metric_name = json_val["metric_name"]
metric_value = json_val["_value"]
parsed_index = json_val.get("parsed_index")
fields = {
f"metric_name:{metric_name}": metric_value,
EventField.FREQUENCY.value: ir.frequency_str,
Expand All @@ -208,47 +199,20 @@ def build_metric_data(
else:
builder.add(EventField.SOURCETYPE, "sc4snmp:metric")

extract_additional_properties(fields, metric_name, metric_value, server_config)
extract_additional_properties(fields, metric_name, metric_value, parsed_index)

builder.add_fields(fields)
return builder.build()


def extract_additional_properties(fields, metric_name, metric_value, server_config):
result = multi_key_lookup(server_config, (enricher_name, enricher_oid_family))
oid_families = result if result else []
any_regex_matched = False

for family in oid_families.keys():
if metric_name.startswith("sc4snmp." + family):
stripped = metric_name[: metric_name.index("_")]
input_text = metric_name[metric_name.index("_") + 1 :] # noqa: E203
def extract_additional_properties(fields, metric_name, metric_value, parsed_index):
stripped = metric_name[: metric_name.index("_")]
del fields["metric_name:" + metric_name]
fields["metric_name:" + stripped] = metric_value

entries = multi_key_lookup(
oid_families, (family, enricher_additional_varbinds)
)
if entries:
regex_entries = [
entry["regex"] for entry in entries if "regex" in entry
]
for regex in regex_entries:
result = re.match(regex, input_text)
if result:
any_regex_matched = True
for key, value in result.groupdict().items():
fields[key] = value.replace("_", ".")
del fields["metric_name:" + metric_name]
fields["metric_name:" + stripped] = metric_value
break
break

if not any_regex_matched:
stripped = metric_name[: metric_name.rindex("_")]
input_text = metric_name[metric_name.rindex("_") + 1 :] # noqa: E203

fields["index_number"] = input_text
del fields["metric_name:" + metric_name]
fields["metric_name:" + stripped] = metric_value
if parsed_index:
for key, value in parsed_index.items():
fields[key] = value


def build_error_data(
Expand Down
14 changes: 0 additions & 14 deletions splunk_connect_for_snmp_poller/manager/task_utilities.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,6 @@ async def snmp_get_handler(
ir,
additional_metric_fields,
var_binds,
server_config,
):
"""
Perform the SNMP Get for an oid,
Expand Down Expand Up @@ -261,7 +260,6 @@ async def snmp_get_handler(
index,
ir,
additional_metric_fields,
server_config,
one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
mib_enricher=mib_enricher,
)
Expand All @@ -278,7 +276,6 @@ async def snmp_get_handler(
index,
ir,
additional_metric_fields,
server_config,
one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
is_error=is_error,
)
Expand Down Expand Up @@ -333,7 +330,6 @@ def _any_walk_failure_happened(
is_metric,
ir,
additional_metric_fields,
server_config,
var_binds,
):
is_error, result = prepare_error_message(
Expand All @@ -349,7 +345,6 @@ def _any_walk_failure_happened(
index,
ir,
additional_metric_fields,
server_config,
one_time_flag=one_time_flag,
is_error=is_error,
)
Expand Down Expand Up @@ -390,7 +385,6 @@ async def snmp_bulk_handler(
ir,
additional_metric_fields,
var_binds,
server_config,
):
"""
Perform the SNMP Bulk for an array of oids
Expand Down Expand Up @@ -426,7 +420,6 @@ async def snmp_bulk_handler(
index,
ir,
additional_metric_fields,
server_config,
one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
mib_enricher=mib_enricher,
)
Expand All @@ -443,7 +436,6 @@ async def snmp_bulk_handler(
index,
ir,
additional_metric_fields,
server_config,
one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
is_error=is_error,
)
Expand All @@ -464,7 +456,6 @@ async def walk_handler(
one_time_flag,
ir,
additional_metric_fields,
server_config,
):
"""
Perform the SNMP Walk for oid end with *,
Expand Down Expand Up @@ -493,7 +484,6 @@ async def walk_handler(
is_metric,
ir,
additional_metric_fields,
server_config,
var_binds,
):
if OnetimeFlag.is_a_walk(one_time_flag):
Expand All @@ -509,7 +499,6 @@ async def walk_handler(
index,
ir,
additional_metric_fields,
server_config,
one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
)
if OnetimeFlag.is_a_walk(one_time_flag):
Expand Down Expand Up @@ -567,7 +556,6 @@ async def walk_handler_with_enricher(
one_time_flag,
ir,
additional_metric_fields,
server_config,
):
"""
Perform the SNMP Walk for oid end with *,
Expand Down Expand Up @@ -597,7 +585,6 @@ async def walk_handler_with_enricher(
is_metric,
ir,
additional_metric_fields,
server_config,
var_binds,
):
break
Expand All @@ -620,7 +607,6 @@ async def walk_handler_with_enricher(
index,
ir,
additional_metric_fields,
server_config,
one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
)

Expand Down
3 changes: 0 additions & 3 deletions splunk_connect_for_snmp_poller/manager/tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@ async def get_snmp_data(
one_time_flag,
ir,
additional_metric_fields,
server_config,
):
if var_binds:
try:
Expand All @@ -86,7 +85,6 @@ async def get_snmp_data(
ir,
additional_metric_fields,
var_binds,
server_config,
)
except Exception as e:
logger.exception(f"Error occurred while calling {handler.__name__}(): {e}")
Expand Down Expand Up @@ -172,7 +170,6 @@ async def snmp_polling_async(
one_time_flag,
ir,
additional_metric_fields,
server_config,
]
get_bulk_specific_parameters = [mongo_connection, enricher_presence]
try:
Expand Down
62 changes: 7 additions & 55 deletions tests/test_additional_data_extraction.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,70 +22,22 @@

class TestAdditionalDataExtraction(TestCase):
def test_data_extraction(self):
server_config = {
"enricher": {
"oidFamily": {
"TCP-MIB": {
"additionalVarBinds": [
{
"regex": "(?P<IP_one>[0-9]+_[0-9]+_[0-9]+_[0-9]+)_(?P<port>[0-9]+)_(?P<IP_two>[0-9]+_[0-9]+_[0-9]+_[0-9]+)_(?P<index_number>[0-9]+)", # noqa: E501
}
]
},
"IF-MIB": {
"existingVarBinds": [
{"ifDescr": "interface_desc"},
{"ifPhysAddress": "MAC_address"},
],
},
"UDP-MIB": {
"additionalVarBinds": [
{
"regex": '(?P<protocol_version_one>ipv4)_"(?P<IP_one>[0-9]+_[0-9]+_[0-9]+_[0-9]+)"_(?P<port_one>[0-9]+)_(?P<protocol_version_two>ipv4)_"(?P<IP_two>[0-9]+_[0-9]+_[0-9]+_[0-9]+)"_(?P<index_number>[0-9]+)_(?P<port_two>[0-9]+)', # noqa: E501
}
]
},
}
}
}

fields = {
"metric_name:sc4snmp.TCP-MIB.tcpConnLocalPort_192_168_0_1_161_127_0_0_1_5": "1111"
}
fields2 = {"metric_name:sc4snmp.IF-MIB.ifInErrors_2_1_asdad_23": "173127"}
fields3 = {
'metric_name:sc4snmp.UDP-MIB.udpEndpointProcess_ipv4_"0_0_0_0"_111_ipv4_"0_0_0_0"_0_13348': "123"

parsed_index = {
"test1": "value1",
"test2": "value2",
}

extract_additional_properties(
fields,
"sc4snmp.TCP-MIB.tcpConnLocalPort_192_168_0_1_161_127_0_0_1_5",
"1111",
server_config,
)

extract_additional_properties(
fields2, "sc4snmp.IF-MIB.ifInErrors_2_1_asdad_23", "173127", server_config
parsed_index,
)

extract_additional_properties(
fields3,
'sc4snmp.UDP-MIB.udpEndpointProcess_ipv4_"0_0_0_0"_111_ipv4_"0_0_0_0"_0_13348',
"123",
server_config,
)

self.assertEqual(fields["IP_one"], "192.168.0.1")
self.assertEqual(fields["port"], "161")
self.assertEqual(fields["IP_two"], "127.0.0.1")
self.assertEqual(fields["index_number"], "5")

self.assertEqual(fields2["index_number"], "23")

self.assertEqual(fields3["protocol_version_one"], "ipv4")
self.assertEqual(fields3["IP_one"], "0.0.0.0")
self.assertEqual(fields3["port_one"], "111")
self.assertEqual(fields3["protocol_version_two"], "ipv4")
self.assertEqual(fields3["IP_two"], "0.0.0.0")
self.assertEqual(fields3["index_number"], "0")
self.assertEqual(fields3["port_two"], "13348")
self.assertEqual(fields["test1"], "value1")
self.assertEqual(fields["test2"], "value2")

0 comments on commit 86611d3

Please sign in to comment.