fix: add check for if walk (#171)

* fix: add fix for multiple IF-MIB calls

* fix: delete redundant log

* fix: add loggers

* fix: change the sequence of IF-MIB walk and full walk

* fix: change true from string to boolean

* fix: cleanup loggers

* fix: add different way of loading server_config

* fix: change comparision to 0

splunk_connect_for_snmp_poller/manager/poller.py

@@ -40,6 +40,7 @@
 from splunk_connect_for_snmp_poller.manager.validator.inventory_validator import (
     DYNAMIC_PROFILE,
 )
+from splunk_connect_for_snmp_poller.manager.variables import onetime_if_walk
 from splunk_connect_for_snmp_poller.mongo import WalkedHostsRepository
 from splunk_connect_for_snmp_poller.utilities import (
     file_was_modified,
@@ -135,8 +136,8 @@ def __check_inventory(self):
                         self._server_config["profiles"],
                     )
                     if entry_key not in self._jobs_map:
-                        self.check_if_new_host_was_added(entry_key, ir, new_enricher)
                         self.process_new_job(entry_key, ir, profiles)
+                        self.check_if_new_host_was_added(entry_key, ir, new_enricher)
                     else:
                         self.update_schedule_for_changed_conf(entry_key, ir, profiles)
 
@@ -150,10 +151,11 @@ def __check_inventory(self):
     def check_if_new_host_was_added(self, host_key, inventory_record, new_enricher):
         ir_host = return_database_id(host_key)
         if self._old_enricher != {}:
-            logger.info(f"New host: {ir_host}")
-            self.__add_enricher_to_a_host(
-                new_enricher, copy.deepcopy(inventory_record), True
-            )
+            if not self._mongo.first_time_walk_was_initiated(ir_host, onetime_if_walk):
+                logger.debug(f"New host added: {ir_host}")
+                self.__add_enricher_to_a_host(
+                    new_enricher, copy.deepcopy(inventory_record), True
+                )
 
     def run_enricher_changed_check(self, new_enricher, inventory_hosts_with_snmp_data):
         logger.info(
@@ -296,7 +298,7 @@ def __start_realtime_scheduler_task(self):
             automatic_onetime_task,
             self._mongo,
             self.__get_splunk_indexes(),
-            self._server_config,
+            self._args.config,
         )
 
     def add_device_for_profile_matching(self, device: InventoryRecord):

splunk_connect_for_snmp_poller/manager/poller_utilities.py

@@ -39,8 +39,13 @@
 from splunk_connect_for_snmp_poller.manager.variables import (
     enricher_if_mib,
     enricher_oid_family,
+    onetime_walk,
+)
+from splunk_connect_for_snmp_poller.utilities import (
+    OnetimeFlag,
+    multi_key_lookup,
+    parse_config_file,
 )
-from splunk_connect_for_snmp_poller.utilities import OnetimeFlag, multi_key_lookup
 
 logger = logging.getLogger(__name__)
 
@@ -58,11 +63,12 @@ def _should_process_current_line(inventory_record: dict):
 
 
 def iterate_through_unwalked_hosts_scheduler(
-    server_config, splunk_indexes, mongo_connection
+    config_location, splunk_indexes, mongo_connection
 ):
     logger.debug("Executing iterate_through_unwalked_hosts_scheduler")
     profile = OidConstant.UNIVERSAL_BASE_OID
     unwalked_hosts = mongo_connection.get_all_unwalked_hosts()
+    server_config = parse_config_file(config_location)
     for unwalked_host in unwalked_hosts:
         inventory_record = InventoryRecord(
             unwalked_host["host"],
@@ -163,7 +169,9 @@ def _extract_sys_uptime_instance(
 
 
 def _walk_info(mongo_collection, host, current_sys_up_time):
-    host_already_walked = mongo_collection.first_time_walked_was_initiated(host) != 0
+    host_already_walked = (
+        mongo_collection.first_time_walk_was_initiated(host, onetime_walk) != 0
+    )
     logger.info(f"host_already_walked: {host_already_walked}")
     should_do_walk = not host_already_walked
     if host_already_walked:
@@ -219,12 +227,12 @@ def automatic_realtime_job(
 def automatic_onetime_task(
     mongo_collection,
     splunk_indexes,
-    server_config,
+    config_location,
 ):
     job_thread = threading.Thread(
         target=iterate_through_unwalked_hosts_scheduler,
         args=[
-            server_config,
+            config_location,
             splunk_indexes,
             mongo_collection,
         ],
@@ -277,9 +285,7 @@ def automatic_realtime_task(
                 sys_up_time,
             )
             if should_do_walk:
-                mongo_collection.update_walked_host(
-                    db_host_id, {"walked_first_time": True}
-                )
+                mongo_collection.update_walked_host(db_host_id, {onetime_walk: True})
     except Exception:
         logger.exception("Error during automatic_realtime_task")
 
@@ -318,7 +324,7 @@ def _update_enricher_config_with_ifmib(
         inventory_host,
         server_config,
         splunk_indexes,
-        None,
+        OnetimeFlag.FIRST_WALK.value,
     )
 
 

splunk_connect_for_snmp_poller/manager/task_utilities.py

@@ -46,6 +46,7 @@
     extract_network_interface_data_from_walk,
 )
 from splunk_connect_for_snmp_poller.manager.static.mib_enricher import MibEnricher
+from splunk_connect_for_snmp_poller.manager.variables import onetime_if_walk
 from splunk_connect_for_snmp_poller.utilities import OnetimeFlag
 
 logger = get_task_logger(__name__)
@@ -485,7 +486,7 @@ async def walk_handler(
             additional_metric_fields,
             var_binds,
         ):
-            if one_time_flag:
+            if OnetimeFlag.is_a_walk(one_time_flag):
                 error_in_one_time_walk = True
             break
         else:
@@ -508,6 +509,7 @@ async def walk_handler(
             f"{host}:{port}",
             ir,
         )
+    logger.info(f"Walk finished for {host} profile={profile}")
 
 
 def extract_data_to_mongo(host, port, mongo_connection, var_binds):
@@ -530,6 +532,9 @@ def extract_data_to_mongo(host, port, mongo_connection, var_binds):
 def process_one_time_flag(
     one_time_flag, error_in_one_time_walk, mongo_connection, host, ir
 ):
+    logger.info(
+        f"process_one_time_flag {one_time_flag} {error_in_one_time_walk} {host}"
+    )
     if one_time_flag == OnetimeFlag.FIRST_WALK.value and error_in_one_time_walk:
         mongo_connection.add_onetime_walk_result(host, ir.version, ir.community)
     if one_time_flag == OnetimeFlag.AFTER_FAIL.value and not error_in_one_time_walk:
@@ -557,7 +562,6 @@ async def walk_handler_with_enricher(
     e.g. 1.3.6.1.2.1.1.9.*,
     which queries the infos correlated to all the oids that underneath the prefix before the *, e.g. 1.3.6.1.2.1.1.9
     """
-    error_in_one_time_walk = False
     merged_result = []
     merged_result_metric = []
     merged_result_non_metric = []
@@ -583,8 +587,6 @@ async def walk_handler_with_enricher(
             additional_metric_fields,
             var_binds,
         ):
-            if OnetimeFlag.is_a_walk(one_time_flag):
-                error_in_one_time_walk = True
             break
         else:
             result, is_metric = await get_translated_string(
@@ -607,16 +609,10 @@ async def walk_handler_with_enricher(
                 additional_metric_fields,
                 one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
             )
-    if OnetimeFlag.is_a_walk(one_time_flag):
-        process_one_time_flag(
-            one_time_flag,
-            error_in_one_time_walk,
-            mongo_connection,
-            f"{host}:{port}",
-            ir,
-        )
 
-    logger.info(f"Walk finished for {host} profile={ir.profile}")
+    logger.info(f"Walk finished for {host} profile={profile}")
+    if merged_result:
+        mongo_connection.update_walked_host(f"{host}:{port}", {onetime_if_walk: True})
     processed_result = extract_network_interface_data_from_walk(enricher, merged_result)
     additional_enricher_varbinds = (
         extract_network_interface_data_from_additional_config(enricher)

splunk_connect_for_snmp_poller/manager/tasks.py

@@ -211,11 +211,26 @@ async def snmp_polling_async(
             # Perform SNNP WALK for oid end with *
             if ir.profile[-1] == "*":
                 logger.info("Executing SNMP WALK for %s profile=%s", host, ir.profile)
-                if ir.profile == OidConstant.IF_MIB:
+                if ir.profile == OidConstant.IF_MIB or (
+                    enricher_presence and one_time_flag == OnetimeFlag.AFTER_FAIL.value
+                ):
+                    logger.debug(
+                        "Executing SNMP small WALK for %s profile=%s",
+                        host,
+                        OidConstant.IF_MIB,
+                    )
                     await walk_handler_with_enricher(
-                        ir.profile, server_config, mongo_connection, *static_parameters
+                        OidConstant.IF_MIB,
+                        server_config,
+                        mongo_connection,
+                        *static_parameters,
+                    )
+                if ir.profile == OidConstant.UNIVERSAL_BASE_OID:
+                    logger.debug(
+                        "Executing SNMP big WALK for %s profile=%s",
+                        host,
+                        ir.profile,
                     )
-                else:
                     await walk_handler(ir.profile, mongo_connection, *static_parameters)
             # Perform SNNP GET for an oid
             else:

splunk_connect_for_snmp_poller/manager/variables.py

@@ -17,3 +17,5 @@
 enricher_additional_varbinds = "additionalVarBinds"
 enricher_oid_family = "oidFamily"
 enricher_if_mib = "IF-MIB"
+onetime_walk = "walked_first_time"
+onetime_if_walk = "ifmib_walked_first_time"

splunk_connect_for_snmp_poller/mongo.py

@@ -108,8 +108,8 @@ def is_connected(self):
     def contains_host(self, host):
         return self._walked_hosts.find({"_id": host}).count()
 
-    def first_time_walked_was_initiated(self, host):
-        return self._walked_hosts.find({"_id": host, "walked_first_time": True}).count()
+    def first_time_walk_was_initiated(self, host, flag_name):
+        return self._walked_hosts.find({"_id": host, flag_name: True}).count()
 
     def add_host(self, host):
         try:
@@ -143,6 +143,7 @@ def clear(self):
         self._walked_hosts.remove()
 
     def update_walked_host(self, host, element):
+        logger.debug(f"Updating walked_hosts for host {host} with = {element}")
         self._walked_hosts.find_one_and_update(
             {"_id": host},
             {"$set": element},