Skip to content
This repository has been archived by the owner on Dec 17, 2021. It is now read-only.

Commit

Permalink
encapsulated hec sending request + test + correct pycov coverage report
Browse files Browse the repository at this point in the history
  • Loading branch information
@okuzhel
okuzhel committed Oct 4, 2021
1 parent 105b784 commit 0c2faf1
Show file tree
Hide file tree
Showing 7 changed files with 238 additions and 143 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build-test-release.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ jobs:
run: |
# shellcheck disable=SC1090
source "$HOME/.poetry/env"
poetry run pytest --cov=./ --cov-report=xml --junitxml=test-results/junit.xml
poetry run pytest --cov=./splunk_connect_for_snmp_poller --cov-report=xml --junitxml=test-results/junit.xml
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v1
with:
Expand Down
159 changes: 95 additions & 64 deletions poetry.lock
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ asgiref = "^3.4.1"
backoff = "^1.11.1"

[tool.poetry.dev-dependencies]
#pytest-splunk-addon = "^1.4.0"
pytest = "*"
pytest = "^6.2.5"
responses = "^0.14.0"
pytest-cov = "^2.12.1"
jsoncomment = "^0.4.2"
pre-commit = "^2.14.0"
Expand Down
81 changes: 45 additions & 36 deletions splunk_connect_for_snmp_poller/manager/hec_sender.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,24 +30,52 @@
logger = get_logger(__name__)


class HecSender:
def __init__(self, metrics_endpoint, logs_endpoint):
logger.debug(f"[-] logs : {logs_endpoint}, metrics : {metrics_endpoint}")
self.metrics_endpoint = metrics_endpoint
self.logs_endpoint = logs_endpoint

def send_hec_request(self, is_metric: bool, data):
if is_metric:
return self.send_metric_request(data)
else:
return self.send_event_request(data)

def send_event_request(self, data):
return HecSender.send_request(self.logs_endpoint, data)

def send_metric_request(self, data):
return HecSender.send_request(self.metrics_endpoint, data)

@staticmethod
def send_request(endpoint, data):
try:
logger.debug("+++++++++endpoint+++++++++\n%s", endpoint)
response = requests.post(url=endpoint, json=data, timeout=60)
logger.debug("Response code is %s", response.status_code)
logger.debug("Response is %s", response.text)
return response
except requests.ConnectionError as e:
logger.error(
f"Connection error when sending data to HEC index - {data['index']}: {e}"
)


def post_data_to_splunk_hec(
host,
logs_endpoint,
metrics_endpoint,
hec_sender: HecSender,
variables_binds,
host,
is_metric,
index,
ir,
ir: InventoryRecord,
additional_metric_fields,
one_time_flag=False,
mib_enricher=None,
):
logger.debug(f"[-] logs : {logs_endpoint}, metrics : {metrics_endpoint}")

if is_metric:
logger.debug(f"+++++++++metric index: {index['metric_index']} +++++++++")
post_metric_data(
metrics_endpoint,
logger.debug(f"metric index: {index['metric_index']}")
data = build_metric_data(
host,
variables_binds,
index["metric_index"],
Expand All @@ -56,20 +84,20 @@ def post_data_to_splunk_hec(
mib_enricher,
)
else:
logger.debug(f"*********event index: {index['event_index']} ********")
post_event_data(
logs_endpoint,
logger.debug(f"event index: {index['event_index']}")
data = build_event_data(
host,
variables_binds,
index,
one_time_flag,
mib_enricher,
)
hec_sender.send_hec_request(is_metric, data)


# TODO Discuss the format of event data payload
def post_event_data(
endpoint, host, variables_binds, indexes, one_time_flag=False, mib_enricher=None
def build_event_data(
host, variables_binds, indexes, one_time_flag=False, mib_enricher=None
):
variables_binds = prepare_variable_binds(mib_enricher, variables_binds)

Expand All @@ -82,17 +110,7 @@ def post_event_data(
builder.add(EventField.INDEX, indexes["event_index"])
builder.add(EventField.SOURCETYPE, EventType.ERROR.value)

data = builder.build()

try:
logger.debug("+++++++++endpoint+++++++++\n%s", endpoint)
response = requests.post(url=endpoint, json=data, timeout=60)
logger.debug("Response code is %s", response.status_code)
logger.debug("Response is %s", response.text)
except requests.ConnectionError as e:
logger.error(
f"Connection error when sending data to HEC index - {data['index']}: {e}"
)
return builder.build()


def init_builder_with_common_data(current_time, host, index) -> EventBuilder:
Expand Down Expand Up @@ -144,8 +162,7 @@ def _enrich_event_data(mib_enricher: MibEnricher, variables_binds: dict) -> str:
return non_metric_result


def post_metric_data(
endpoint,
def build_metric_data(
host,
variables_binds,
index,
Expand All @@ -170,15 +187,7 @@ def post_metric_data(
builder.add(EventField.EVENT, EventType.METRIC.value)
builder.add_fields(fields)

data = builder.build()

try:
logger.debug("-----endpoint------\n%s", endpoint)
response = requests.post(url=endpoint, json=data, timeout=60)
logger.debug("Response code is %s", response.status_code)
logger.debug("Response is %s", response.text)
except requests.ConnectionError as e:
logger.error(f"Connection error when sending data to HEC index - {index}: {e}")
return builder.build()


def _enrich_metric_data(
Expand Down
53 changes: 17 additions & 36 deletions splunk_connect_for_snmp_poller/manager/task_utilities.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,14 +200,13 @@ async def snmp_get_handler(
mongo_connection,
enricher_presence,
snmp_engine,
hec_sender,
auth_data,
context_data,
host,
port,
mib_server_url,
index,
otel_logs_url,
otel_metrics_url,
one_time_flag,
ir,
additional_metric_fields,
Expand Down Expand Up @@ -236,9 +235,7 @@ async def snmp_get_handler(
mib_server_url, [varbind], return_multimetric
)
post_data_to_splunk_hec(
host,
otel_logs_url,
otel_metrics_url,
hec_sender,
result,
is_metric,
index,
Expand All @@ -253,9 +250,7 @@ async def snmp_get_handler(
)
if is_error:
post_data_to_splunk_hec(
host,
otel_logs_url,
otel_metrics_url,
hec_sender,
result,
False, # fail during bulk so sending to event index
index,
Expand Down Expand Up @@ -304,13 +299,12 @@ def _any_failure_happened(


def _any_walk_failure_happened(
hec_sender,
error_indication,
error_status,
error_index,
host,
index,
otel_logs_url,
otel_metrics_url,
one_time_flag,
is_metric,
ir,
Expand All @@ -323,9 +317,8 @@ def _any_walk_failure_happened(

if is_error:
post_data_to_splunk_hec(
hec_sender,
host,
otel_logs_url,
otel_metrics_url,
result,
is_metric,
index,
Expand Down Expand Up @@ -359,14 +352,13 @@ async def snmp_bulk_handler(
mongo_connection,
enricher_presence,
snmp_engine,
hec_sender,
auth_data,
context_data,
host,
port,
mib_server_url,
index,
otel_logs_url,
otel_metrics_url,
one_time_flag,
ir,
additional_metric_fields,
Expand Down Expand Up @@ -399,9 +391,8 @@ async def snmp_bulk_handler(
mib_server_url, [varbind], return_multimetric
)
post_data_to_splunk_hec(
hec_sender,
host,
otel_logs_url,
otel_metrics_url,
result,
is_metric,
index,
Expand All @@ -416,9 +407,8 @@ async def snmp_bulk_handler(
)
if is_error:
post_data_to_splunk_hec(
hec_sender,
host,
otel_logs_url,
otel_metrics_url,
result,
False, # fail during bulk so sending to event index
index,
Expand All @@ -431,14 +421,13 @@ async def snmp_bulk_handler(
async def walk_handler(
profile,
snmp_engine,
hec_sender,
auth_data,
context_data,
host,
port,
mib_server_url,
index,
otel_logs_url,
otel_metrics_url,
one_time_flag,
ir,
additional_metric_fields,
Expand All @@ -459,13 +448,12 @@ async def walk_handler(
):
is_metric = False
if _any_walk_failure_happened(
hec_sender,
errorIndication,
errorStatus,
errorIndex,
host,
index,
otel_logs_url,
otel_metrics_url,
one_time_flag,
is_metric,
ir,
Expand All @@ -476,9 +464,8 @@ async def walk_handler(
else:
result, is_metric = await get_translated_string(mib_server_url, varBinds)
post_data_to_splunk_hec(
hec_sender,
host,
otel_logs_url,
otel_metrics_url,
result,
is_metric,
index,
Expand All @@ -493,14 +480,13 @@ async def walk_handler_with_enricher(
enricher,
mongo_connection,
snmp_engine,
hec_sender,
auth_data,
context_data,
host,
port,
mib_server_url,
index,
otel_logs_url,
otel_metrics_url,
one_time_flag,
ir,
additional_metric_fields,
Expand All @@ -523,13 +509,12 @@ async def walk_handler_with_enricher(
):
is_metric = False
if _any_walk_failure_happened(
hec_sender,
errorIndication,
errorStatus,
errorIndex,
host,
index,
otel_logs_url,
otel_metrics_url,
ir,
additional_metric_fields,
one_time_flag,
Expand Down Expand Up @@ -561,19 +546,17 @@ async def walk_handler_with_enricher(
)
post_walk_data_to_splunk_arguments = [
host,
otel_logs_url,
otel_metrics_url,
index,
one_time_flag,
ir,
additional_metric_fields,
mib_enricher,
]
_post_walk_data_to_splunk(
merged_result_metric, True, *post_walk_data_to_splunk_arguments
hec_sender, merged_result_metric, True, *post_walk_data_to_splunk_arguments
)
_post_walk_data_to_splunk(
merged_result_non_metric, False, *post_walk_data_to_splunk_arguments
hec_sender, merged_result_non_metric, False, *post_walk_data_to_splunk_arguments
)


Expand Down Expand Up @@ -628,11 +611,10 @@ def _return_mib_enricher_for_walk(


def _post_walk_data_to_splunk(
hec_sender,
result_list,
is_metric,
host,
otel_logs_url,
otel_metrics_url,
index,
one_time_flag,
ir,
Expand All @@ -641,9 +623,8 @@ def _post_walk_data_to_splunk(
):
for result in result_list:
post_data_to_splunk_hec(
hec_sender,
host,
otel_logs_url,
otel_metrics_url,
result,
is_metric,
index,
Expand Down
Loading

0 comments on commit 0c2faf1

Please sign in to comment.