Skip to content
This repository has been archived by the owner on Dec 17, 2021. It is now read-only.

feat: suffix parsing #187

Merged
merged 5 commits into from
Nov 4, 2021
Merged

feat: suffix parsing #187

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
232c21e
feat: suffix parsing
weliasz Nov 4, 2021
af42f6a
fix: build fix
weliasz Nov 4, 2021
6711f3f
fix: build fix
weliasz Nov 4, 2021
c2b21c6
fix: PR comments
weliasz Nov 4, 2021
0b6aa47
fix: fix
weliasz Nov 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 9 additions & 45 deletions splunk_connect_for_snmp_poller/manager/hec_sender.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
# limitations under the License.
# ########################################################################
import json
import re
import time

import requests
Expand All @@ -27,12 +26,6 @@
)
from splunk_connect_for_snmp_poller.manager.data.inventory_record import InventoryRecord
from splunk_connect_for_snmp_poller.manager.static.mib_enricher import MibEnricher
from splunk_connect_for_snmp_poller.manager.variables import (
enricher_additional_varbinds,
enricher_name,
enricher_oid_family,
)
from splunk_connect_for_snmp_poller.utilities import multi_key_lookup

logger = get_logger(__name__)

Expand Down Expand Up @@ -77,7 +70,6 @@ def post_data_to_splunk_hec(
index,
ir: InventoryRecord,
additional_metric_fields,
server_config,
one_time_flag=False,
mib_enricher=None,
is_error=False,
Expand All @@ -93,7 +85,6 @@ def post_data_to_splunk_hec(
index["metric_index"],
ir,
additional_metric_fields,
server_config,
one_time_flag=one_time_flag,
mib_enricher=mib_enricher,
)
Expand Down Expand Up @@ -184,13 +175,13 @@ def build_metric_data(
index,
ir: InventoryRecord,
additional_metric_fields,
server_config,
one_time_flag=False,
mib_enricher=None,
):
json_val = json.loads(variables_binds)
metric_name = json_val["metric_name"]
metric_value = json_val["_value"]
parsed_index = json_val.get("parsed_index")
fields = {
f"metric_name:{metric_name}": metric_value,
EventField.FREQUENCY.value: ir.frequency_str,
Expand All @@ -208,47 +199,20 @@ def build_metric_data(
else:
builder.add(EventField.SOURCETYPE, "sc4snmp:metric")

extract_additional_properties(fields, metric_name, metric_value, server_config)
extract_additional_properties(fields, metric_name, metric_value, parsed_index)

builder.add_fields(fields)
return builder.build()


def extract_additional_properties(fields, metric_name, metric_value, server_config):
result = multi_key_lookup(server_config, (enricher_name, enricher_oid_family))
oid_families = result if result else []
any_regex_matched = False

for family in oid_families.keys():
if metric_name.startswith("sc4snmp." + family):
stripped = metric_name[: metric_name.index("_")]
input_text = metric_name[metric_name.index("_") + 1 :] # noqa: E203
def extract_additional_properties(fields, metric_name, metric_value, parsed_index):
stripped = metric_name[: metric_name.index("_")]
del fields["metric_name:" + metric_name]
fields["metric_name:" + stripped] = metric_value

entries = multi_key_lookup(
oid_families, (family, enricher_additional_varbinds)
)
if entries:
regex_entries = [
entry["regex"] for entry in entries if "regex" in entry
]
for regex in regex_entries:
result = re.match(regex, input_text)
if result:
any_regex_matched = True
for key, value in result.groupdict().items():
fields[key] = value.replace("_", ".")
del fields["metric_name:" + metric_name]
fields["metric_name:" + stripped] = metric_value
break
break

if not any_regex_matched:
stripped = metric_name[: metric_name.rindex("_")]
input_text = metric_name[metric_name.rindex("_") + 1 :] # noqa: E203

fields["index_number"] = input_text
del fields["metric_name:" + metric_name]
fields["metric_name:" + stripped] = metric_value
if parsed_index:
for key, value in parsed_index.items():
fields[key] = value


def build_error_data(
Expand Down
15 changes: 1 addition & 14 deletions splunk_connect_for_snmp_poller/manager/task_utilities.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,6 @@ async def snmp_get_handler(
ir,
additional_metric_fields,
var_binds,
server_config,
):
"""
Perform the SNMP Get for an oid,
Expand Down Expand Up @@ -261,7 +260,6 @@ async def snmp_get_handler(
index,
ir,
additional_metric_fields,
server_config,
one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
mib_enricher=mib_enricher,
)
Expand All @@ -278,7 +276,6 @@ async def snmp_get_handler(
index,
ir,
additional_metric_fields,
server_config,
one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
is_error=is_error,
)
Expand Down Expand Up @@ -333,7 +330,6 @@ def _any_walk_failure_happened(
is_metric,
ir,
additional_metric_fields,
server_config,
var_binds,
):
is_error, result = prepare_error_message(
Expand All @@ -349,7 +345,6 @@ def _any_walk_failure_happened(
index,
ir,
additional_metric_fields,
server_config,
one_time_flag=one_time_flag,
is_error=is_error,
)
Expand Down Expand Up @@ -390,7 +385,6 @@ async def snmp_bulk_handler(
ir,
additional_metric_fields,
var_binds,
server_config,
):
"""
Perform the SNMP Bulk for an array of oids
Expand Down Expand Up @@ -426,7 +420,6 @@ async def snmp_bulk_handler(
index,
ir,
additional_metric_fields,
server_config,
one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
mib_enricher=mib_enricher,
)
Expand All @@ -443,7 +436,6 @@ async def snmp_bulk_handler(
index,
ir,
additional_metric_fields,
server_config,
one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
is_error=is_error,
)
Expand Down Expand Up @@ -493,14 +485,13 @@ async def walk_handler(
is_metric,
ir,
additional_metric_fields,
server_config,
var_binds,
):
if OnetimeFlag.is_a_walk(one_time_flag):
error_in_one_time_walk = True
break
else:
result, is_metric = await get_translated_string(mib_server_url, var_binds)
omrozowicz-splunk marked this conversation as resolved.
Show resolved Hide resolved
result = await get_translated_string(mib_server_url, var_binds)
post_data_to_splunk_hec(
hec_sender,
host,
Expand All @@ -509,7 +500,6 @@ async def walk_handler(
index,
ir,
additional_metric_fields,
server_config,
one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
)
if OnetimeFlag.is_a_walk(one_time_flag):
Expand Down Expand Up @@ -567,7 +557,6 @@ async def walk_handler_with_enricher(
one_time_flag,
ir,
additional_metric_fields,
server_config,
):
"""
Perform the SNMP Walk for oid end with *,
Expand Down Expand Up @@ -597,7 +586,6 @@ async def walk_handler_with_enricher(
is_metric,
ir,
additional_metric_fields,
server_config,
var_binds,
):
break
Expand All @@ -620,7 +608,6 @@ async def walk_handler_with_enricher(
index,
ir,
additional_metric_fields,
server_config,
one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
)

Expand Down
3 changes: 0 additions & 3 deletions splunk_connect_for_snmp_poller/manager/tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@ async def get_snmp_data(
one_time_flag,
ir,
additional_metric_fields,
server_config,
):
if var_binds:
try:
Expand All @@ -86,7 +85,6 @@ async def get_snmp_data(
ir,
additional_metric_fields,
var_binds,
server_config,
)
except Exception as e:
logger.exception(f"Error occurred while calling {handler.__name__}(): {e}")
Expand Down Expand Up @@ -172,7 +170,6 @@ async def snmp_polling_async(
one_time_flag,
ir,
additional_metric_fields,
server_config,
]
get_bulk_specific_parameters = [mongo_connection, enricher_presence]
try:
Expand Down
62 changes: 7 additions & 55 deletions tests/test_additional_data_extraction.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,70 +22,22 @@

class TestAdditionalDataExtraction(TestCase):
def test_data_extraction(self):
server_config = {
"enricher": {
"oidFamily": {
"TCP-MIB": {
"additionalVarBinds": [
{
"regex": "(?P<IP_one>[0-9]+_[0-9]+_[0-9]+_[0-9]+)_(?P<port>[0-9]+)_(?P<IP_two>[0-9]+_[0-9]+_[0-9]+_[0-9]+)_(?P<index_number>[0-9]+)", # noqa: E501
}
]
},
"IF-MIB": {
"existingVarBinds": [
{"ifDescr": "interface_desc"},
{"ifPhysAddress": "MAC_address"},
],
},
"UDP-MIB": {
"additionalVarBinds": [
{
"regex": '(?P<protocol_version_one>ipv4)_"(?P<IP_one>[0-9]+_[0-9]+_[0-9]+_[0-9]+)"_(?P<port_one>[0-9]+)_(?P<protocol_version_two>ipv4)_"(?P<IP_two>[0-9]+_[0-9]+_[0-9]+_[0-9]+)"_(?P<index_number>[0-9]+)_(?P<port_two>[0-9]+)', # noqa: E501
}
]
},
}
}
}

fields = {
"metric_name:sc4snmp.TCP-MIB.tcpConnLocalPort_192_168_0_1_161_127_0_0_1_5": "1111"
}
fields2 = {"metric_name:sc4snmp.IF-MIB.ifInErrors_2_1_asdad_23": "173127"}
fields3 = {
'metric_name:sc4snmp.UDP-MIB.udpEndpointProcess_ipv4_"0_0_0_0"_111_ipv4_"0_0_0_0"_0_13348': "123"

parsed_index = {
"test1": "value1",
"test2": "value2",
}

extract_additional_properties(
fields,
"sc4snmp.TCP-MIB.tcpConnLocalPort_192_168_0_1_161_127_0_0_1_5",
"1111",
server_config,
)

extract_additional_properties(
fields2, "sc4snmp.IF-MIB.ifInErrors_2_1_asdad_23", "173127", server_config
parsed_index,
)

extract_additional_properties(
fields3,
'sc4snmp.UDP-MIB.udpEndpointProcess_ipv4_"0_0_0_0"_111_ipv4_"0_0_0_0"_0_13348',
"123",
server_config,
)

self.assertEqual(fields["IP_one"], "192.168.0.1")
self.assertEqual(fields["port"], "161")
self.assertEqual(fields["IP_two"], "127.0.0.1")
self.assertEqual(fields["index_number"], "5")

self.assertEqual(fields2["index_number"], "23")

self.assertEqual(fields3["protocol_version_one"], "ipv4")
self.assertEqual(fields3["IP_one"], "0.0.0.0")
self.assertEqual(fields3["port_one"], "111")
self.assertEqual(fields3["protocol_version_two"], "ipv4")
self.assertEqual(fields3["IP_two"], "0.0.0.0")
self.assertEqual(fields3["index_number"], "0")
self.assertEqual(fields3["port_two"], "13348")
self.assertEqual(fields["test1"], "value1")
self.assertEqual(fields["test2"], "value2")