Fix: missing test case for regression imported_from in the arbiter fo…

…r modules. Was related to #1595.

modules/dummy_arbiter/module.py

@@ -49,7 +49,7 @@ def get_instance(plugin):
 # Just print some stuff
 class Dummy_arbiter(BaseModule):
     def __init__(self, mod_conf):
-        BaseModule.__init__(mod_conf)
+        BaseModule.__init__(self, mod_conf)
 
     # Called by Arbiter to say 'let's prepare yourself guy'
     def init(self):

shinken/daemons/arbiterdaemon.py

@@ -336,39 +336,9 @@ def load_config_file(self):
         # Call modules that manage this read configuration pass
         self.hook_point('read_configuration')
 
-        # Now we ask for configuration modules if they
-        # got items for us
-        for inst in self.modules_manager.instances:
-            # TODO : clean
-            if hasattr(inst, 'get_objects'):
-                _t = time.time()
-                try:
-                    r = inst.get_objects()
-                except Exception, exp:
-                    logger.error("Instance %s raised an exception %s. Log and continue to run",
-                                 inst.get_name(), str(exp))
-                    output = cStringIO.StringIO()
-                    traceback.print_exc(file=output)
-                    logger.error("Back trace of this remove: %s", output.getvalue())
-                    output.close()
-                    continue
-                statsmgr.incr('hook.get-objects', time.time() - _t)
-                types_creations = self.conf.types_creations
-                for k in types_creations:
-                    (cls, clss, prop, dummy) = types_creations[k]
-                    if prop in r:
-                        for x in r[prop]:
-                            # test if raw_objects[k] are already set - if not, add empty array
-                            if k not in raw_objects:
-                                raw_objects[k] = []
-                            # put the imported_from property if the module is not already setting
-                            # it so we know where does this object came from
-                            if 'imported_from' not in x:
-                                x['imported_from'] = 'module:%s' % inst.get_name()
-                            # now append the object
-                            raw_objects[k].append(x)
-                        logger.debug("Added %i objects to %s from module %s",
-                                     len(r[prop]), k, inst.get_name())
+        # Call modules get_objects() to load new objects from them
+        # (example modules: glpi, mongodb, dummy_arbiter)
+        self.load_modules_configuration_objects(raw_objects)
 
         # Resume standard operations ###
         self.conf.create_objects(raw_objects)
@@ -518,6 +488,43 @@ def load_config_file(self):
         logger.info("Configuration Loaded")
 
 
+    def load_modules_configuration_objects(self, raw_objects):
+        # Now we ask for configuration modules if they
+        # got items for us
+        for inst in self.modules_manager.instances:
+            # TODO : clean
+            if hasattr(inst, 'get_objects'):
+                _t = time.time()
+                try:
+                    r = inst.get_objects()
+                except Exception, exp:
+                    logger.error("Instance %s raised an exception %s. Log and continue to run",
+                                 inst.get_name(), str(exp))
+                    output = cStringIO.StringIO()
+                    traceback.print_exc(file=output)
+                    logger.error("Back trace of this remove: %s", output.getvalue())
+                    output.close()
+                    continue
+                statsmgr.incr('hook.get-objects', time.time() - _t)
+                types_creations = self.conf.types_creations
+                for k in types_creations:
+                    (cls, clss, prop, dummy) = types_creations[k]
+                    if prop in r:
+                        for x in r[prop]:
+                            # test if raw_objects[k] are already set - if not, add empty array
+                            if k not in raw_objects:
+                                raw_objects[k] = []
+                            # put the imported_from property if the module is not already setting
+                            # it so we know where does this object came from
+                            if 'imported_from' not in x:
+                                x['imported_from'] = 'module:%s' % inst.get_name()
+                            # now append the object
+                            raw_objects[k].append(x)
+                        logger.debug("Added %i objects to %s from module %s",
+                                     len(r[prop]), k, inst.get_name())
+
+
+
     def launch_analyse(self):
 
         logger.info("We are doing an statistic analysis on the dump file %s", self.analyse)

shinken/modulesmanager.py

@@ -78,6 +78,7 @@ def load_and_init(self):
         self.load()
         self.get_instances()
 
+
     @classmethod
     def try_best_load(cls, name, package=None):
         try:
@@ -119,6 +120,7 @@ def try_very_bad_load(cls, mod_dir):
             if prev_module is not None:  # and restore it after we have loaded our one (or not)
                 sys.modules['module'] = prev_module
 
+
     @classmethod
     def try_load(cls, mod_name, mod_dir=None):
         msg = ''
@@ -137,6 +139,7 @@ def try_load(cls, mod_name, mod_dir=None):
             logger.info(msg, mod_name)
         return mod
 
+
     # Try to import the requested modules ; put the imported modules in self.imported_modules.
     # The previous imported modules, if any, are cleaned before.
     def load(self):

test/etc/missing_imported_from_module_property/commands.cfg

@@ -0,0 +1,30 @@
+define command{
+    command_name    check-host-alive
+    command_line    $USER1$/test_hostcheck.pl --type=$ARG1$ --failchance=2% --previous-state=$HOSTSTATE$ --state-duration=$HOSTDURATIONSEC$ --hostname $HOSTNAME$
+}
+define command{
+    command_name    check-host-alive-parent
+    command_line    $USER1$/test_hostcheck.pl --type=$ARG1$ --failchance=2% --previous-state=$HOSTSTATE$ --state-duration=$HOSTDURATIONSEC$ --parent-state=$ARG2$ --hostname $HOSTNAME$
+}
+define command{
+    command_name    notify-host
+    #command_line    sleep 1 && /bin/true
+    command_line    $USER1$/notifier.pl --hostname $HOSTNAME$ --notificationtype $NOTIFICATIONTYPE$ --hoststate $HOSTSTATE$ --hostoutput $HOSTOUTPUT$ --longdatetime $LONGDATETIME$ --hostattempt $HOSTATTEMPT$ --hoststatetype $HOSTSTATETYPE$
+}
+define command{
+    command_name    notify-service
+    command_line    $USER1$/notifier.pl --hostname $HOSTNAME$ --servicedesc $SERVICEDESC$ --notificationtype $NOTIFICATIONTYPE$ --servicestate $SERVICESTATE$ --serviceoutput $SERVICEOUTPUT$ --longdatetime $LONGDATETIME$ --serviceattempt $SERVICEATTEMPT$ --servicestatetype $SERVICESTATETYPE$
+    #command_line    sleep 1 && /bin/true
+}
+define command{
+    command_name    check_service
+    command_line    $USER1$/test_servicecheck.pl --type=$ARG1$ --failchance=5% --previous-state=$SERVICESTATE$ --state-duration=$SERVICEDURATIONSEC$ --total-critical-on-host=$TOTALHOSTSERVICESCRITICAL$ --total-warning-on-host=$TOTALHOSTSERVICESWARNING$ --hostname $HOSTNAME$ --servicedesc $SERVICEDESC$ --custom $_SERVICECUSTNAME$
+}
+define command{
+  command_name eventhandler
+  command_line    $USER1$/test_eventhandler.pl $SERVICESTATE$ $SERVICESTATETYPE$ $SERVICEATTEMPT$
+}
+define command{
+  command_name    special_macro
+  command_line	  $USER1$/nothing $ARG1$
+}

test/etc/missing_imported_from_module_property/test_specific.cfg

@@ -0,0 +1,26 @@
+define arbiter {
+    arbiter_name    arbiter-master
+    #host_name       node1       ; CHANGE THIS if you have several Arbiters
+    address         localhost   ; DNS name or IP
+    port            7770
+    spare           0      
+    modules	    forarbiter
+
+    # Enable https or not
+    use_ssl               0
+    # enable certificate/hostname check, will avoid man in the middle attacks
+    hard_ssl_name_check   0
+
+    ## Uncomment these lines in a HA architecture so the master and slaves know
+    ## how long they may wait for each other.
+    #timeout              3   ; Ping timeout
+    #data_timeout         120 ; Data send timeout
+    #max_check_attempts   3   ; If ping fails N or more, then the node is dead
+    #check_interval       60  ; Ping node every N seconds
+}
+
+
+define module{
+     module_name    forarbiter
+     module_type    dummy_arbiter
+}

test/etc/shinken_missing_imported_from_module_property.cfg

@@ -0,0 +1,121 @@
+accept_passive_host_checks=1
+accept_passive_service_checks=1
+additional_freshness_latency=15
+admin_email=shinken@localhost
+admin_pager=shinken@localhost
+auto_reschedule_checks=0
+auto_rescheduling_interval=30
+auto_rescheduling_window=180
+cached_host_check_horizon=15
+cached_service_check_horizon=15
+cfg_file=standard/hosts.cfg
+cfg_file=standard/services.cfg
+cfg_file=standard/contacts.cfg
+cfg_file=missing_imported_from_module_property/commands.cfg
+cfg_file=missing_imported_from_module_property/test_specific.cfg
+cfg_file=standard/timeperiods.cfg
+cfg_file=standard/hostgroups.cfg
+cfg_file=standard/servicegroups.cfg
+cfg_file=standard/shinken-specific.cfg
+check_external_commands=1
+check_for_orphaned_hosts=1
+check_for_orphaned_services=1
+check_host_freshness=0
+check_result_path=var/spool/checkresults
+check_result_reaper_frequency=10
+check_service_freshness=1
+command_check_interval=-1
+command_file=var/shinken.cmd
+daemon_dumps_core=0
+date_format=iso8601
+debug_file=var/shinken.debug
+debug_level=112
+debug_verbosity=1
+enable_embedded_perl=0
+enable_environment_macros=1
+enable_event_handlers=1
+enable_flap_detection=0
+enable_notifications=1
+enable_predictive_host_dependency_checks=1
+enable_predictive_service_dependency_checks=1
+event_broker_options=-1
+event_handler_timeout=30
+execute_host_checks=1
+execute_service_checks=1
+external_command_buffer_slots=4096
+high_host_flap_threshold=20
+high_service_flap_threshold=20
+host_check_timeout=30
+host_freshness_check_interval=60
+host_inter_check_delay_method=s
+illegal_macro_output_chars=`~\$&|'"<>
+illegal_object_name_chars=`~!\$%^&*|'"<>?,()=
+interval_length=60
+lock_file=var/shinken.pid
+log_archive_path=var/archives
+log_event_handlers=1
+log_external_commands=1
+log_file=var/shinken.log
+log_host_retries=1
+log_initial_states=0
+log_notifications=1
+log_passive_checks=1
+log_rotation_method=d
+log_service_retries=1
+low_host_flap_threshold=5
+low_service_flap_threshold=5
+max_check_result_file_age=3600
+max_check_result_reaper_time=30
+max_concurrent_checks=0
+max_debug_file_size=1000000
+max_host_check_spread=30
+max_service_check_spread=30
+shinken_group=shinken
+shinken_user=shinken
+notification_timeout=30
+object_cache_file=var/objects.cache
+obsess_over_hosts=0
+obsess_over_services=0
+ocsp_timeout=5
+#p1_file=/tmp/test_shinken/plugins/p1.pl
+p1_file=/usr/local/shinken/bin/p1.pl
+passive_host_checks_are_soft=0
+perfdata_timeout=5
+precached_object_file=var/objects.precache
+process_performance_data=1
+resource_file=resource.cfg
+retain_state_information=1
+retained_contact_host_attribute_mask=0
+retained_contact_service_attribute_mask=0
+retained_host_attribute_mask=0
+retained_process_host_attribute_mask=0
+retained_process_service_attribute_mask=0
+retained_service_attribute_mask=0
+retention_update_interval=60
+service_check_timeout=60
+service_freshness_check_interval=60
+service_inter_check_delay_method=s
+service_interleave_factor=s
+##shinken_group=shinken
+##shinken_user=shinken
+#shinken_group=shinken
+#shinken_user=shinken
+sleep_time=0.25
+soft_state_dependencies=0
+state_retention_file=var/retention.dat
+status_file=var/status.dat
+status_update_interval=5
+temp_file=tmp/shinken.tmp
+temp_path=var/tmp
+translate_passive_host_checks=0
+use_aggressive_host_checking=0
+use_embedded_perl_implicitly=0
+use_large_installation_tweaks=0
+use_regexp_matching=0
+use_retained_program_state=1
+use_retained_scheduling_info=1
+use_syslog=0
+use_true_regexp_matching=0
+enable_problem_impacts_states_change=1
+no_event_handlers_during_downtimes=0
+modules_dir=module_missing_imported_from_module_property

test/module_missing_imported_from_module_property/dummy_arbiter/module.py

@@ -0,0 +1,85 @@
+#!/usr/bin/python
+
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2009-2014:
+#    Gabes Jean, naparuba@gmail.com
+#    Gerhard Lausser, Gerhard.Lausser@consol.de
+#    Gregory Starck, g.starck@gmail.com
+#    Hartmut Goebel, h.goebel@goebel-consult.de
+#
+# This file is part of Shinken.
+#
+# Shinken is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Shinken is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Shinken.  If not, see <http://www.gnu.org/licenses/>.
+
+# This Class is an example of an Arbiter module
+# Here for the configuration phase AND running one
+
+import time
+
+from shinken.basemodule import BaseModule
+from shinken.external_command import ExternalCommand
+from shinken.log import logger
+
+properties = {
+    'daemons': ['arbiter'],
+    'type': 'dummy_arbiter',
+    'external': True,
+    }
+
+
+# called by the plugin manager to get a broker
+def get_instance(plugin):
+    logger.info("[Dummy Arbiter] Get a Dummy arbiter module for plugin %s", plugin.get_name())
+    instance = Dummy_arbiter(plugin)
+    return instance
+
+
+# Just print some stuff
+class Dummy_arbiter(BaseModule):
+    def __init__(self, mod_conf):
+        BaseModule.__init__(self, mod_conf)
+
+    # Called by Arbiter to say 'let's prepare yourself guy'
+    def init(self):
+        logger.info("[Dummy Arbiter] Initialization of the dummy arbiter module")
+        #self.return_queue = self.properties['from_queue']
+
+
+    # Ok, main function that is called in the CONFIGURATION phase
+    def get_objects(self):
+        logger.info("[Dummy Arbiter] Ask me for objects to return")
+        r = {'hosts': []}
+        h = {'name': 'dummy host from dummy arbiter module',
+             'register': '0',
+             }
+
+        r['hosts'].append(h)
+        r['hosts'].append({
+                            'host_name': "dummyhost1",
+                            'use': 'linux-server',
+                            'address': 'localhost'
+                            })
+        logger.info("[Dummy Arbiter] Returning to Arbiter the hosts: %s", str(r))
+
+        return r
+
+    def hook_late_configuration(self, conf):
+        logger.info("[Dummy Arbiter] Dummy in hook late config")
+
+    def do_loop_turn(self):
+        logger.info("[Dummy Arbiter] Raise a external command as example")
+        e = ExternalCommand('Viva la revolution')
+        self.from_q.put(e)
+        time.sleep(1)