Skip to content
This repository has been archived by the owner on Dec 17, 2021. It is now read-only.

feat: force inventory refresh #149

Merged
merged 7 commits into from
Oct 15, 2021
Merged

feat: force inventory refresh #149

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d09e65b
feat: force inventory refresh
weliasz Oct 13, 2021
db1a90a
fix: build fix
weliasz Oct 13, 2021
bed9a60
fix: PR comments
weliasz Oct 13, 2021
0b7a0fa
fix: PR comments
weliasz Oct 13, 2021
764ce07
fix: fetch frequency only when required
weliasz Oct 14, 2021
8726135
fix: build fix
weliasz Oct 14, 2021
3cc0021
Merge branch 'develop' of https://github.com/splunk/splunk-connect-fo…
weliasz Oct 14, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion splunk_connect_for_snmp_poller/manager/poller.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ def __init__(self, args, server_config):
self._local_snmp_engine = SnmpEngine()
self._unmatched_devices = {}
self._lock = threading.Lock()
self._force_refresh = False

def force_inventory_refresh(self):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we change it and request in a few different scheduled functions, can't we run into any race condition?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Boolean assignment is considered threadsafe in Python.
self._force_refresh = False - is the first statement in refreshing fragment of code so if other thread sets the flag to true it will be picked up during next iteration

self._force_refresh = True

def __get_splunk_indexes(self):
return {
Expand Down Expand Up @@ -88,7 +92,9 @@ def __check_inventory(self):
)

# update job when either inventory changes or config changes
if server_config_modified or inventory_config_modified:
if server_config_modified or inventory_config_modified or self._force_refresh:
logger.info("Refreshing inventory and config")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe add force_refresh state to that log to have more information

self._force_refresh = False
inventory_hosts = set()
profiles = get_profiles(self._server_config)
for ir in parse_inventory_file(self._args.inventory, profiles):
Expand Down Expand Up @@ -195,6 +201,8 @@ def __start_realtime_scheduler_task(self):
self.__get_splunk_indexes(),
self._server_config,
self._local_snmp_engine,
self.force_inventory_refresh,
False,
)

schedule.every(self._args.matching_task_frequency).seconds.do(
Expand All @@ -208,6 +216,8 @@ def __start_realtime_scheduler_task(self):
self.__get_splunk_indexes(),
self._server_config,
self._local_snmp_engine,
self.force_inventory_refresh,
True,
)

def add_device_for_profile_matching(self, device: InventoryRecord):
Expand Down
22 changes: 13 additions & 9 deletions splunk_connect_for_snmp_poller/manager/poller_utilities.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
import csv
import logging.config
import threading
from pathlib import Path

import schedule
from pysnmp.hlapi import ObjectIdentity, ObjectType, UdpTransportTarget, getCmd
Expand Down Expand Up @@ -62,9 +61,8 @@ def onetime_task(inventory_record: InventoryRecord, server_config, splunk_indexe
return schedule.CancelJob


def refresh_inventory(inventory_file_path):
Path(inventory_file_path).touch()

def refresh_inventory(force_inventory_refresh):
force_inventory_refresh()
return schedule.CancelJob


Expand Down Expand Up @@ -163,6 +161,8 @@ def automatic_realtime_job(
splunk_indexes,
server_config,
local_snmp_engine,
force_inventory_refresh,
initial_walk,
):
job_thread = threading.Thread(
target=automatic_realtime_task,
Expand All @@ -172,6 +172,8 @@ def automatic_realtime_job(
splunk_indexes,
server_config,
local_snmp_engine,
force_inventory_refresh,
initial_walk,
],
)
job_thread.start()
Expand All @@ -183,6 +185,8 @@ def automatic_realtime_task(
splunk_indexes,
server_config,
local_snmp_engine,
force_inventory_refresh,
initial_walk,
):
try:
for inventory_record in parse_inventory_file(
Expand All @@ -208,11 +212,11 @@ def automatic_realtime_task(
server_config,
splunk_indexes,
)
# touch inventory file after 2 min to trigger reloading of inventory with new walk data
schedule.every(2).minutes.do(
refresh_inventory,
inventory_file_path,
)
if not initial_walk:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we schedule automatic_realtime_task with False in __start_realtime_scheduler_task
wouldn't we schedule new jobs every time it will run?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The feature of forcing inventory refreshing is supposed to be called when new walk for device was conducted and in case sysDescr has changed we would like run the matching again. When we start the application automatic_realtime_task is called to schedule first walk for device and there is no need to force inventory reloading.

# force inventory reloading after 2 min with new walk data
schedule.every(2).minutes.do(
refresh_inventory, force_inventory_refresh
)
_update_mongo(
all_walked_hosts_collection,
db_host_id,
Expand Down