diff --git a/THANKS b/THANKS index 1cd328c783..98426ef0dc 100644 --- a/THANKS +++ b/THANKS @@ -76,4 +76,6 @@ To David Laval for his graphite templates on the linux checks To nerocide for his bug report about bad event handler catch To Mathieu.md for multiple configuration file updates To raphaeltr for a typo patch on a monitoring pack -To Jan Ulferts for his patch about env macros fix on notifications \ No newline at end of file +To Jan Ulferts for his patch about env macros fix on notifications +To Rémi Sauvat for his patch about dashboard automatic widget refresh +To chris81 for his patch for graphite API URLs diff --git a/bin/shinken-arbiter b/bin/shinken-arbiter index 92695d5bd8..c70f353729 100755 --- a/bin/shinken-arbiter +++ b/bin/shinken-arbiter @@ -91,6 +91,10 @@ parser.add_option("-p", "--profile", parser.add_option("-a", "--analyse", dest="analyse", help="Dump an analyse statistics file, for support") +parser.add_option("-m", "--migrate", + dest="migrate", + help="Migrate the raw configuration read from the arbier to another module. --> VERY EXPERIMENTAL!") + opts, args = parser.parse_args() diff --git a/bin/stop_all.sh b/bin/stop_all.sh index 17ced2f3c0..b10ae59f75 100755 --- a/bin/stop_all.sh +++ b/bin/stop_all.sh @@ -24,9 +24,10 @@ DIR="$(cd $(dirname "$0"); pwd)" -"$DIR"/stop_scheduler.sh + "$DIR"/stop_poller.sh "$DIR"/stop_reactionner.sh "$DIR"/stop_broker.sh "$DIR"/stop_receiver.sh +"$DIR"/stop_scheduler.sh "$DIR"/stop_arbiter.sh diff --git a/etc/shinken-specific.cfg b/etc/shinken-specific.cfg index f9bcf1fc7e..c703e7d0dc 100755 --- a/etc/shinken-specific.cfg +++ b/etc/shinken-specific.cfg @@ -66,7 +66,7 @@ define arbiter { # - WS_Arbiter = WebService for pushing results to the arbiter # - Collectd = Wait collectd data, and take perfdata for # services from it. - modules + modules #modules CommandFile, Mongodb, NSCA, VMWare_auto_linking, WS_Arbiter, Collectd, Landscape ## Uncomment these lines in a HA architecture so the master and slaves know diff --git a/share/templates/graphite/check_cpu.graph b/share/templates/graphite/check_cpu.graph index 2fd6882da1..64089160f1 100644 --- a/share/templates/graphite/check_cpu.graph +++ b/share/templates/graphite/check_cpu.graph @@ -1 +1 @@ -$uri/render/?width=586&height=308&_salt=1333718798.689&target=alias(legendValue($host.$service.'user'%2C%22last%22)%2C%22User%22)&target=alias(legendValue($host.$service.'sys'%2C%22last%22)%2C%22Sys%22)&target=alias(legendValue($host.$service.'softirq'%2C%22last%22)%2C%22SoftIRQ%22)&target=alias(legendValue($host.$service.'nice'%2C%22last%22)%2C%22Nice%22)&target=alias(legendValue($host.$service.'irq'%2C%22last%22)%2C%22IRQ%22)&target=alias(legendValue($host.$service.'iowait'%2C%22last%22)%2C%22I%2FO%20Wait%22)&target=alias(legendValue($host.$service.'idle'%2C%22last%22)%2C%22Idle%22)&fgcolor=000000&bgcolor=FFFFFF)&areaMode=stacked&yMax=100 +${uri}render/?width=586&height=308&_salt=1333718798.689&target=alias(legendValue(${host}.${service}.'user'%2C%22last%22)%2C%22User%22)&target=alias(legendValue(${host}.${service}.'sys'%2C%22last%22)%2C%22Sys%22)&target=alias(legendValue(${host}.${service}.'softirq'%2C%22last%22)%2C%22SoftIRQ%22)&target=alias(legendValue(${host}.${service}.'nice'%2C%22last%22)%2C%22Nice%22)&target=alias(legendValue(${host}.${service}.'irq'%2C%22last%22)%2C%22IRQ%22)&target=alias(legendValue(${host}.${service}.'iowait'%2C%22last%22)%2C%22I%2FO%20Wait%22)&target=alias(legendValue(${host}.${service}.'idle'%2C%22last%22)%2C%22Idle%22)&fgcolor=000000&bgcolor=FFFFFF)&areaMode=stacked&yMax=100 diff --git a/share/templates/graphite/check_http.graph b/share/templates/graphite/check_http.graph index 0fb019cc45..1fb16ad568 100644 --- a/share/templates/graphite/check_http.graph +++ b/share/templates/graphite/check_http.graph @@ -1,2 +1,2 @@ -$uri/render/?width=586&height=308&fgcolor=000000&bgcolor=FFFFFF&title=HTTP%20Response%20time&vtitle=Time%20s&target=legendValue(alias($host.$service.time%2C"Time")%2C"last") -$uri/render/?width=586&height=308&fgcolor=000000&bgcolor=FFFFFF&title=HTTP%20Response%20size&vtitle=Size%20B&target=legendValue(alias($host.$service.size%2C"Size")%2C"last") +${uri}render/?width=586&height=308&fgcolor=000000&bgcolor=FFFFFF&title=HTTP%20Response%20time&vtitle=Time%20s&target=legendValue(alias(${host}.${service}.time%2C"Time")%2C"last") +${uri}render/?width=586&height=308&fgcolor=000000&bgcolor=FFFFFF&title=HTTP%20Response%20size&vtitle=Size%20B&target=legendValue(alias(${host}.${service}.size%2C"Size")%2C"last") diff --git a/share/templates/graphite/check_linux_cpu.graph b/share/templates/graphite/check_linux_cpu.graph index 208ef8023f..cf8b1a466c 100644 --- a/share/templates/graphite/check_linux_cpu.graph +++ b/share/templates/graphite/check_linux_cpu.graph @@ -1 +1 @@ -$uri/render/?width=586&height=308&target=alias(legendValue($host.$service.cpu_prct_used%2C%22last%22)%2C%22CPU load%22)&fgcolor=000000&bgcolor=FFFFFF)&yMin=0 +${uri}render/?width=586&height=308&target=alias(legendValue(${host}.${service}.cpu_prct_used%2C%22last%22)%2C%22CPU load%22)&fgcolor=000000&bgcolor=FFFFFF)&yMin=0 diff --git a/share/templates/graphite/check_linux_load.graph b/share/templates/graphite/check_linux_load.graph index 67b9578255..b26b9fb520 100644 --- a/share/templates/graphite/check_linux_load.graph +++ b/share/templates/graphite/check_linux_load.graph @@ -1 +1 @@ -$uri/render/?width=586&height=308&target=legendValue(alias($host.$service.load_1_min%2C%22Load 1 minute%22)%2C%22last%22)&title=Load%20on%20$host&yMin=0&target=legendValue(alias($host.$service.load_5_min%2C%22Load 5 minutes%22)%2C%22last%22)&target=legendValue(alias($host.$service.load_15_min%2C%22Load 15 minutes%22)%2C%22last%22)&fgcolor=000000&bgcolor=FFFFFF& +${uri}render/?width=586&height=308&target=legendValue(alias(${host}.${service}.load_1_min%2C%22Load 1 minute%22)%2C%22last%22)&title=Load%20on%20${host}&yMin=0&target=legendValue(alias(${host}.${service}.load_5_min%2C%22Load 5 minutes%22)%2C%22last%22)&target=legendValue(alias(${host}.${service}.load_15_min%2C%22Load 15 minutes%22)%2C%22last%22)&fgcolor=000000&bgcolor=FFFFFF& diff --git a/share/templates/graphite/check_linux_memory.graph b/share/templates/graphite/check_linux_memory.graph index e20402fae6..e16ef8fdc1 100644 --- a/share/templates/graphite/check_linux_memory.graph +++ b/share/templates/graphite/check_linux_memory.graph @@ -1 +1 @@ -$uri/render/?width=586&height=308&target=legendValue(alias(color(scale($host.$service.ram_used%2C1024)%2C%22%23FF0000%22)%2C%22RAM%22)%2C%22last%22)&target=color(legendValue(alias(scale($host.$service.swap_used%2C1024)%2C%22Swap%22)%2C%22last%22)%2C%22%230000FF%22)&fgcolor=000000&bgcolor=FFFFFF +${uri}render/?width=586&height=308&target=legendValue(alias(color(scale(${host}.${service}.ram_used%2C1024)%2C%22%23FF0000%22)%2C%22RAM%22)%2C%22last%22)&target=color(legendValue(alias(scale(${host}.${service}.swap_used%2C1024)%2C%22Swap%22)%2C%22last%22)%2C%22%230000FF%22)&fgcolor=000000&bgcolor=FFFFFF diff --git a/share/templates/graphite/check_load.graph b/share/templates/graphite/check_load.graph index 08f35f0cc9..81e8a23657 100644 --- a/share/templates/graphite/check_load.graph +++ b/share/templates/graphite/check_load.graph @@ -1 +1 @@ -$uri/render/?width=586&height=308&target=legendValue(alias($host.$service.load1%2C%22Load 1 minute%22)%2C%22last%22)&title=Load%20on%20$host&yMin=0&target=legendValue(alias($host.$service.load5%2C%22Load 5 minutes%22)%2C%22last%22)&target=legendValue(alias($host.$service.load15%2C%22Load 15 minutes%22)%2C%22last%22)&fgcolor=000000&bgcolor=FFFFFF& +${uri}render/?width=586&height=308&target=legendValue(alias(${host}.${service}.load1%2C%22Load 1 minute%22)%2C%22last%22)&title=Load%20on%20${host}&yMin=0&target=legendValue(alias(${host}.${service}.load5%2C%22Load 5 minutes%22)%2C%22last%22)&target=legendValue(alias(${host}.${service}.load15%2C%22Load 15 minutes%22)%2C%22last%22)&fgcolor=000000&bgcolor=FFFFFF& diff --git a/share/templates/graphite/check_mem.graph b/share/templates/graphite/check_mem.graph index 14e32cc634..272982ef92 100644 --- a/share/templates/graphite/check_mem.graph +++ b/share/templates/graphite/check_mem.graph @@ -1 +1 @@ -$uri/render/?width=586&height=308&target=legendValue(alias(color(scale($host.$service.USED%2C1024)%2C%22%23FF0000%22)%2C%22Used%22)%2C%22last%22)&target=color(legendValue(alias(scale($host.$service.CACHES%2C1024)%2C%22Caches%22)%2C%22last%22)%2C%22%230000FF%22)&target=legendValue(color(alias(legendValue(scale(diffSeries($host.$service.TOTAL%2C$host.$service.CACHES%2C$host.$service.USED)%2C1024)%2C%22last%22)%2C%22Free%22)%2C%22%2300FF00%22)%2C%22last%22)&fgcolor=000000&bgcolor=FFFFFF&areaMode=stacked +${uri}render/?width=586&height=308&target=legendValue(alias(color(scale(${host}.${service}.USED%2C1024)%2C%22%23FF0000%22)%2C%22Used%22)%2C%22last%22)&target=color(legendValue(alias(scale(${host}.${service}.CACHES%2C1024)%2C%22Caches%22)%2C%22last%22)%2C%22%230000FF%22)&target=legendValue(color(alias(legendValue(scale(diffSeries(${host}.${service}.TOTAL%2C${host}.${service}.CACHES%2C${host}.${service}.USED)%2C1024)%2C%22last%22)%2C%22Free%22)%2C%22%2300FF00%22)%2C%22last%22)&fgcolor=000000&bgcolor=FFFFFF&areaMode=stacked diff --git a/share/templates/graphite/check_ping.graph b/share/templates/graphite/check_ping.graph index 4170a553d1..3c6749bc06 100644 --- a/share/templates/graphite/check_ping.graph +++ b/share/templates/graphite/check_ping.graph @@ -1,2 +1,2 @@ -$uri/render/?width=586&height=308&target=legendValue(alias($host.$service.rta%2C%22Response%20Time%22)%2C%22last%22)&title=Response%20Time%20on%20$host&yMin=0 -$uri/render/?width=586&height=308&target=legendValue(alias($host.$service.pl%2C%22Paquet%20lost%20percentage%22)%2C%22last%22)&title=Packet%20Lost%20Percentage%20on%20$host&yMin=0&yMax=100 +${uri}render/?width=586&height=308&target=legendValue(alias(${host}.${service}.rta%2C%22Response%20Time%22)%2C%22last%22)&title=Response%20Time%20on%20${host}&yMin=0 +${uri}render/?width=586&height=308&target=legendValue(alias(${host}.${service}.pl%2C%22Paquet%20lost%20percentage%22)%2C%22last%22)&title=Packet%20Lost%20Percentage%20on%20${host}&yMin=0&yMax=100 diff --git a/share/templates/graphite/dashboard/check_http.graph b/share/templates/graphite/dashboard/check_http.graph index 34c43c65fc..7490ca6819 100644 --- a/share/templates/graphite/dashboard/check_http.graph +++ b/share/templates/graphite/dashboard/check_http.graph @@ -1 +1 @@ -$uri/render/?width=586&height=308&fgcolor=000000&bgcolor=FFFFFF&title=HTTP%20Response%20time&vtitle=Time%20s&target=legendValue(alias($host.$service.time%2C"Time")%2C"last") +${uri}render/?width=586&height=308&fgcolor=000000&bgcolor=FFFFFF&title=HTTP%20Response%20time&vtitle=Time%20s&target=legendValue(alias(${host}.${service}.time%2C"Time")%2C"last") diff --git a/shinken/daemons/arbiterdaemon.py b/shinken/daemons/arbiterdaemon.py index 51d467a673..9e9a20da6a 100644 --- a/shinken/daemons/arbiterdaemon.py +++ b/shinken/daemons/arbiterdaemon.py @@ -133,15 +133,14 @@ def get_objects_properties(self, table, *properties): # Main Arbiter Class class Arbiter(Daemon): - def __init__(self, config_files, is_daemon, do_replace, verify_only, debug, debug_file, profile=None, analyse=None): + def __init__(self, config_files, is_daemon, do_replace, verify_only, debug, debug_file, profile=None, analyse=None, migrate=None): super(Arbiter, self).__init__('arbiter', config_files[0], is_daemon, do_replace, debug, debug_file) self.config_files = config_files - self.verify_only = verify_only - self.analyse = analyse + self.migrate = migrate self.broks = {} self.is_master = False @@ -160,6 +159,7 @@ def __init__(self, config_files, is_daemon, do_replace, verify_only, debug, debu self.interface = IForArbiter(self) self.conf = Config() + # Use for adding things like broks def add(self, b): if isinstance(b, Brok): @@ -312,6 +312,11 @@ def load_config_file(self): # Manage all post-conf modules self.hook_point('early_configuration') + # Ok here maybe we should stop because we are in a pure migration run + if self.migrate: + print "Migration MODE. Early exiting from configuration relinking phase" + return + # Load all file triggers self.conf.load_triggers() @@ -477,6 +482,47 @@ def launch_analyse(self): f.close() + def go_migrate(self): + print "***********"*5 + print "WARNING : this feature is NOT supported in this version!" + print "***********"*5 + + migration_module_name = self.migrate.strip() + mig_mod = self.conf.modules.find_by_name(migration_module_name) + if not mig_mod: + print "Cannot find the migration module %s. Please configure it" % migration_module_name + sys.exit(2) + + print self.modules_manager.instances + # Ok now all we need is the import module + self.modules_manager.set_modules([mig_mod]) + self.do_load_modules() + print self.modules_manager.instances + if len(self.modules_manager.instances) == 0: + print "Error duringthe initialization of the import module. Bailing out" + sys.exit(2) + print "Configuration migrating in progress..." + mod = self.modules_manager.instances[0] + f = getattr(mod, 'import_objects', None) + if not f or not callable(f): + print "Import module is missing the import_objects function. Bailing out" + sys.exit(2) + + objs = {} + types = ['hosts', 'services', 'commands', 'timeperiods', 'contacts'] + for t in types: + print "New type", t + objs[t] = [] + for i in getattr(self.conf, t): + d = i.get_raw_import_values() + if d: + objs[t].append(d) + f(objs) + # Ok we can exit now + sys.exit(0) + + + # Main loop function def main(self): try: @@ -486,6 +532,10 @@ def main(self): self.load_config_file() + # Maybe we are in a migration phase. If so, we will bailout here + if self.migrate: + self.go_migrate() + # Look if we are enabled or not. If ok, start the daemon mode self.look_for_early_exit() self.do_daemon_init_and_start() diff --git a/shinken/modules/mongodb_generic.py b/shinken/modules/mongodb_generic.py index 7f3e166622..b91826eb87 100644 --- a/shinken/modules/mongodb_generic.py +++ b/shinken/modules/mongodb_generic.py @@ -29,6 +29,10 @@ # This module imports hosts and services configuration from a MySQL Database # Queries for getting hosts and services are pulled from shinken-specific.cfg configuration file. +try: + import uuid +except ImportError: + uuid = None try: from pymongo.connection import Connection @@ -70,21 +74,21 @@ def __init__(self, mod_conf, uri, database): # Called by Arbiter to say 'let's prepare yourself guy' def init(self): - print "[Mongodb Module]: Try to open a Mongodb connection to %s:%s" % (self.uri, self.database) + logger.info("[Mongodb Module]: Try to open a Mongodb connection to %s:%s" % (self.uri, self.database)) try: self.con = Connection(self.uri) self.db = getattr(self.con, self.database) except Exception, e: - print "Mongodb Module: Error %s:" % e + logger.error("Mongodb Module: Error %s:" % e) raise - print "[Mongodb Module]: Connection OK" + logger.info("[Mongodb Module]: Connection OK") ################################ Arbiter part ################################# # Main function that is called in the CONFIGURATION phase def get_objects(self): if not self.db: - print "[Mongodb Module]: Problem during init phase" + logger.error("[Mongodb Module]: Problem during init phase") return {} r = {} @@ -95,7 +99,7 @@ def get_objects(self): cur = getattr(self.db, t).find({'_state': {'$ne': 'disabled'}}) for h in cur: - print "DBG: mongodb: get an ", t, h + #print "DBG: mongodb: get an ", t, h # We remove a mongodb specific property, the _id del h['_id'] # And we add an imported_from property to say it came from @@ -105,6 +109,59 @@ def get_objects(self): return r + + def get_uniq_id(self, t, i): + #by default we will return a very uniq id + u = str(int(uuid.uuid4().int)) + + is_tpl = (getattr(i, 'register', '1') == '0') + if is_tpl: + return 'tpl-%s' % getattr(i, 'name', u) + + lst = {'hosts' : 'host_name', 'commands' : 'command_name', + 'timeperiods' : 'timeperiod_name', + 'contacts' : 'contact_name', + } + if t in lst: + prop = lst[t] + n = getattr(i, prop, None) + if n: + return n + return u + if t == 'services': + return u + + print "Unknown TYPE in migration!" + return u + + + + # Function called by the arbiter so we import the objects in our databases + def import_objects(self, data): + if not self.db: + logger.error("[Mongodb]: error Problem during init phase") + return False + + # Maybe we can't have a good way to have uniq id, if so, bail out + if not uuid: + logger.error("Your python version is too old. Please update to a 2.6 version to use this feature") + return False + + + for t in data: + col = getattr(self.db, t) + print "Saving objects %s" % t + elts = data[t] + for e in elts: + print "Element", e + e['_id'] = self.get_uniq_id(t, e) + col.save(e) + + + return True + + + #################################### WebUI parts ############################ # We will get in the mongodb database the user preference entry, for the 'shinken-global' user diff --git a/shinken/objects/item.py b/shinken/objects/item.py index 26c26c6440..ce90a7093b 100644 --- a/shinken/objects/item.py +++ b/shinken/objects/item.py @@ -369,6 +369,22 @@ def old_properties_names_to_new(self): value = getattr(self, old_name) setattr(self, new_name, value) + # The arbiter is asking us our raw value before all explode or linking + def get_raw_import_values(self): + r = {} + properties = self.__class__.properties.keys() + # Register is not by default in the properties + if not 'register' in properties: + properties.append('register') + + for prop in properties: + if hasattr(self, prop): + v = getattr(self, prop) + print prop, ":", v + r[prop] = v + return r + + def add_downtime(self, downtime): self.downtimes.append(downtime) diff --git a/shinken/objects/timeperiod.py b/shinken/objects/timeperiod.py index bf9b7ac058..1bcbdcd4a9 100644 --- a/shinken/objects/timeperiod.py +++ b/shinken/objects/timeperiod.py @@ -87,6 +87,7 @@ import time import re +import string from item import Item, Items @@ -149,6 +150,23 @@ def get_unresolved_properties_by_inheritance(self, items): for i in self.templates: self.unresolved.extend(i.unresolved) + # Ok timeperiods are a bit different from classic items, because we do not have a real list + # of our raw rpoperties, like if we got february 1 - 15 / 3 for example + def get_raw_import_values(self): + properties = ['timeperiod_name', 'alias', 'use', 'register'] + r = {} + for prop in properties: + if hasattr(self, prop): + v = getattr(self, prop) + print prop, ":", v + r[prop] = v + # Now the unresolved one. The only way to get ride of same key things is to put + # directly the full value as the key + for other in self.unresolved: + r[other] = '' + return r + + def is_time_valid(self, t): if self.has('exclude'): for dr in self.exclude: diff --git a/shinken/webui/plugins_skonf/elements/elements.py b/shinken/webui/plugins_skonf/elements/elements.py index b1d35cf4c4..e2a29cde10 100644 --- a/shinken/webui/plugins_skonf/elements/elements.py +++ b/shinken/webui/plugins_skonf/elements/elements.py @@ -63,6 +63,7 @@ def elements_generic(cls, show_tpls=False): key = keys[t] #if cls.my_type == 'host': # print "HOOK HOSTS" + elts = [cls(i) for i in app.datamgr.get_generics(t, key)] print "GOT elements", elts diff --git a/shinken/webui/plugins_skonf/elements/local_helper.py b/shinken/webui/plugins_skonf/elements/local_helper.py index 53dedba2a6..5438452059 100644 --- a/shinken/webui/plugins_skonf/elements/local_helper.py +++ b/shinken/webui/plugins_skonf/elements/local_helper.py @@ -35,7 +35,10 @@ def find(value, lst, key): for i in lst: v = i.get(key, None) - print 'MAtch with', v, value + try: + print 'MAtch with', v, value + except: + pass if v == value: return i return None @@ -61,7 +64,10 @@ def find_several(lst, elt, prop, key): if v is None: continue v = v.strip() - print 'MAtch with db', v + try: + print 'MAtch with db', v + except: + pass if v in values: res.append(dbi) print "Return find_several::", res