Merge remote-tracking branch 'upstream/master'

THANKS

@@ -76,4 +76,6 @@ To David Laval for his graphite templates on the linux checks
 To nerocide for his bug report about bad event handler catch
 To Mathieu.md for multiple configuration file updates
 To raphaeltr for a typo patch on a monitoring pack
-To Jan Ulferts for his patch about env macros fix on notifications
+To Jan Ulferts for his patch about env macros fix on notifications
+To Rémi Sauvat for his patch about dashboard automatic widget refresh
+To chris81 for his patch for graphite API URLs

bin/shinken-arbiter

@@ -91,6 +91,10 @@ parser.add_option("-p", "--profile",
 parser.add_option("-a", "--analyse",
                   dest="analyse",
                   help="Dump an analyse statistics file, for support")
+parser.add_option("-m", "--migrate",
+                  dest="migrate",
+                  help="Migrate the raw configuration read from the arbier to another module. --> VERY EXPERIMENTAL!")
+
 
 opts, args = parser.parse_args()
 

bin/stop_all.sh

@@ -24,9 +24,10 @@
 
 DIR="$(cd $(dirname "$0"); pwd)"
 
-"$DIR"/stop_scheduler.sh
+
 "$DIR"/stop_poller.sh
 "$DIR"/stop_reactionner.sh
 "$DIR"/stop_broker.sh
 "$DIR"/stop_receiver.sh
+"$DIR"/stop_scheduler.sh
 "$DIR"/stop_arbiter.sh

etc/shinken-specific.cfg

@@ -66,7 +66,7 @@ define arbiter {
     # - WS_Arbiter              = WebService for pushing results to the arbiter
     # - Collectd                = Wait collectd data, and take perfdata for
     #                               services from it.
-    modules      
+    modules    		  
     #modules     CommandFile, Mongodb, NSCA, VMWare_auto_linking, WS_Arbiter, Collectd, Landscape
 
     ## Uncomment these lines in a HA architecture so the master and slaves know

share/templates/graphite/check_cpu.graph

@@ -1 +1 @@
-$uri/render/?width=586&height=308&_salt=1333718798.689&target=alias(legendValue($host.$service.'user'%2C%22last%22)%2C%22User%22)&target=alias(legendValue($host.$service.'sys'%2C%22last%22)%2C%22Sys%22)&target=alias(legendValue($host.$service.'softirq'%2C%22last%22)%2C%22SoftIRQ%22)&target=alias(legendValue($host.$service.'nice'%2C%22last%22)%2C%22Nice%22)&target=alias(legendValue($host.$service.'irq'%2C%22last%22)%2C%22IRQ%22)&target=alias(legendValue($host.$service.'iowait'%2C%22last%22)%2C%22I%2FO%20Wait%22)&target=alias(legendValue($host.$service.'idle'%2C%22last%22)%2C%22Idle%22)&fgcolor=000000&bgcolor=FFFFFF)&areaMode=stacked&yMax=100
+${uri}render/?width=586&height=308&_salt=1333718798.689&target=alias(legendValue(${host}.${service}.'user'%2C%22last%22)%2C%22User%22)&target=alias(legendValue(${host}.${service}.'sys'%2C%22last%22)%2C%22Sys%22)&target=alias(legendValue(${host}.${service}.'softirq'%2C%22last%22)%2C%22SoftIRQ%22)&target=alias(legendValue(${host}.${service}.'nice'%2C%22last%22)%2C%22Nice%22)&target=alias(legendValue(${host}.${service}.'irq'%2C%22last%22)%2C%22IRQ%22)&target=alias(legendValue(${host}.${service}.'iowait'%2C%22last%22)%2C%22I%2FO%20Wait%22)&target=alias(legendValue(${host}.${service}.'idle'%2C%22last%22)%2C%22Idle%22)&fgcolor=000000&bgcolor=FFFFFF)&areaMode=stacked&yMax=100

share/templates/graphite/check_http.graph

@@ -1,2 +1,2 @@
-$uri/render/?width=586&height=308&fgcolor=000000&bgcolor=FFFFFF&title=HTTP%20Response%20time&vtitle=Time%20s&target=legendValue(alias($host.$service.time%2C"Time")%2C"last")
-$uri/render/?width=586&height=308&fgcolor=000000&bgcolor=FFFFFF&title=HTTP%20Response%20size&vtitle=Size%20B&target=legendValue(alias($host.$service.size%2C"Size")%2C"last")
+${uri}render/?width=586&height=308&fgcolor=000000&bgcolor=FFFFFF&title=HTTP%20Response%20time&vtitle=Time%20s&target=legendValue(alias(${host}.${service}.time%2C"Time")%2C"last")
+${uri}render/?width=586&height=308&fgcolor=000000&bgcolor=FFFFFF&title=HTTP%20Response%20size&vtitle=Size%20B&target=legendValue(alias(${host}.${service}.size%2C"Size")%2C"last")

share/templates/graphite/check_linux_cpu.graph

@@ -1 +1 @@
-$uri/render/?width=586&height=308&target=alias(legendValue($host.$service.cpu_prct_used%2C%22last%22)%2C%22CPU load%22)&fgcolor=000000&bgcolor=FFFFFF)&yMin=0
+${uri}render/?width=586&height=308&target=alias(legendValue(${host}.${service}.cpu_prct_used%2C%22last%22)%2C%22CPU load%22)&fgcolor=000000&bgcolor=FFFFFF)&yMin=0

share/templates/graphite/check_linux_load.graph

@@ -1 +1 @@
-$uri/render/?width=586&height=308&target=legendValue(alias($host.$service.load_1_min%2C%22Load 1 minute%22)%2C%22last%22)&title=Load%20on%20$host&yMin=0&target=legendValue(alias($host.$service.load_5_min%2C%22Load 5 minutes%22)%2C%22last%22)&target=legendValue(alias($host.$service.load_15_min%2C%22Load 15 minutes%22)%2C%22last%22)&fgcolor=000000&bgcolor=FFFFFF&
+${uri}render/?width=586&height=308&target=legendValue(alias(${host}.${service}.load_1_min%2C%22Load 1 minute%22)%2C%22last%22)&title=Load%20on%20${host}&yMin=0&target=legendValue(alias(${host}.${service}.load_5_min%2C%22Load 5 minutes%22)%2C%22last%22)&target=legendValue(alias(${host}.${service}.load_15_min%2C%22Load 15 minutes%22)%2C%22last%22)&fgcolor=000000&bgcolor=FFFFFF&

share/templates/graphite/check_linux_memory.graph

@@ -1 +1 @@
-$uri/render/?width=586&height=308&target=legendValue(alias(color(scale($host.$service.ram_used%2C1024)%2C%22%23FF0000%22)%2C%22RAM%22)%2C%22last%22)&target=color(legendValue(alias(scale($host.$service.swap_used%2C1024)%2C%22Swap%22)%2C%22last%22)%2C%22%230000FF%22)&fgcolor=000000&bgcolor=FFFFFF
+${uri}render/?width=586&height=308&target=legendValue(alias(color(scale(${host}.${service}.ram_used%2C1024)%2C%22%23FF0000%22)%2C%22RAM%22)%2C%22last%22)&target=color(legendValue(alias(scale(${host}.${service}.swap_used%2C1024)%2C%22Swap%22)%2C%22last%22)%2C%22%230000FF%22)&fgcolor=000000&bgcolor=FFFFFF

share/templates/graphite/check_load.graph

@@ -1 +1 @@
-$uri/render/?width=586&height=308&target=legendValue(alias($host.$service.load1%2C%22Load 1 minute%22)%2C%22last%22)&title=Load%20on%20$host&yMin=0&target=legendValue(alias($host.$service.load5%2C%22Load 5 minutes%22)%2C%22last%22)&target=legendValue(alias($host.$service.load15%2C%22Load 15 minutes%22)%2C%22last%22)&fgcolor=000000&bgcolor=FFFFFF&
+${uri}render/?width=586&height=308&target=legendValue(alias(${host}.${service}.load1%2C%22Load 1 minute%22)%2C%22last%22)&title=Load%20on%20${host}&yMin=0&target=legendValue(alias(${host}.${service}.load5%2C%22Load 5 minutes%22)%2C%22last%22)&target=legendValue(alias(${host}.${service}.load15%2C%22Load 15 minutes%22)%2C%22last%22)&fgcolor=000000&bgcolor=FFFFFF&

share/templates/graphite/check_mem.graph

@@ -1 +1 @@
-$uri/render/?width=586&height=308&target=legendValue(alias(color(scale($host.$service.USED%2C1024)%2C%22%23FF0000%22)%2C%22Used%22)%2C%22last%22)&target=color(legendValue(alias(scale($host.$service.CACHES%2C1024)%2C%22Caches%22)%2C%22last%22)%2C%22%230000FF%22)&target=legendValue(color(alias(legendValue(scale(diffSeries($host.$service.TOTAL%2C$host.$service.CACHES%2C$host.$service.USED)%2C1024)%2C%22last%22)%2C%22Free%22)%2C%22%2300FF00%22)%2C%22last%22)&fgcolor=000000&bgcolor=FFFFFF&areaMode=stacked
+${uri}render/?width=586&height=308&target=legendValue(alias(color(scale(${host}.${service}.USED%2C1024)%2C%22%23FF0000%22)%2C%22Used%22)%2C%22last%22)&target=color(legendValue(alias(scale(${host}.${service}.CACHES%2C1024)%2C%22Caches%22)%2C%22last%22)%2C%22%230000FF%22)&target=legendValue(color(alias(legendValue(scale(diffSeries(${host}.${service}.TOTAL%2C${host}.${service}.CACHES%2C${host}.${service}.USED)%2C1024)%2C%22last%22)%2C%22Free%22)%2C%22%2300FF00%22)%2C%22last%22)&fgcolor=000000&bgcolor=FFFFFF&areaMode=stacked

share/templates/graphite/check_ping.graph

@@ -1,2 +1,2 @@
-$uri/render/?width=586&height=308&target=legendValue(alias($host.$service.rta%2C%22Response%20Time%22)%2C%22last%22)&title=Response%20Time%20on%20$host&yMin=0
-$uri/render/?width=586&height=308&target=legendValue(alias($host.$service.pl%2C%22Paquet%20lost%20percentage%22)%2C%22last%22)&title=Packet%20Lost%20Percentage%20on%20$host&yMin=0&yMax=100
+${uri}render/?width=586&height=308&target=legendValue(alias(${host}.${service}.rta%2C%22Response%20Time%22)%2C%22last%22)&title=Response%20Time%20on%20${host}&yMin=0
+${uri}render/?width=586&height=308&target=legendValue(alias(${host}.${service}.pl%2C%22Paquet%20lost%20percentage%22)%2C%22last%22)&title=Packet%20Lost%20Percentage%20on%20${host}&yMin=0&yMax=100

share/templates/graphite/dashboard/check_http.graph

@@ -1 +1 @@
-$uri/render/?width=586&height=308&fgcolor=000000&bgcolor=FFFFFF&title=HTTP%20Response%20time&vtitle=Time%20s&target=legendValue(alias($host.$service.time%2C"Time")%2C"last")
+${uri}render/?width=586&height=308&fgcolor=000000&bgcolor=FFFFFF&title=HTTP%20Response%20time&vtitle=Time%20s&target=legendValue(alias(${host}.${service}.time%2C"Time")%2C"last")

shinken/daemons/arbiterdaemon.py

@@ -133,15 +133,14 @@ def get_objects_properties(self, table, *properties):
 # Main Arbiter Class
 class Arbiter(Daemon):
 
-    def __init__(self, config_files, is_daemon, do_replace, verify_only, debug, debug_file, profile=None, analyse=None):
+    def __init__(self, config_files, is_daemon, do_replace, verify_only, debug, debug_file, profile=None, analyse=None, migrate=None):
 
         super(Arbiter, self).__init__('arbiter', config_files[0], is_daemon, do_replace, debug, debug_file)
 
         self.config_files = config_files
-
         self.verify_only = verify_only
-
         self.analyse = analyse
+        self.migrate = migrate
 
         self.broks = {}
         self.is_master = False
@@ -160,6 +159,7 @@ def __init__(self, config_files, is_daemon, do_replace, verify_only, debug, debu
         self.interface = IForArbiter(self)
         self.conf = Config()
 
+
     # Use for adding things like broks
     def add(self, b):
         if isinstance(b, Brok):
@@ -312,6 +312,11 @@ def load_config_file(self):
         # Manage all post-conf modules
         self.hook_point('early_configuration')
 
+        # Ok here maybe we should stop because we are in a pure migration run
+        if self.migrate:
+            print "Migration MODE. Early exiting from configuration relinking phase"
+            return
+
         # Load all file triggers
         self.conf.load_triggers()
 
@@ -477,6 +482,47 @@ def launch_analyse(self):
         f.close()
 
 
+    def go_migrate(self):
+        print "***********"*5
+        print "WARNING : this feature is NOT supported in this version!"
+        print "***********"*5
+
+        migration_module_name = self.migrate.strip()
+        mig_mod = self.conf.modules.find_by_name(migration_module_name)
+        if not mig_mod:
+            print "Cannot find the migration module %s. Please configure it" % migration_module_name
+            sys.exit(2)
+
+        print self.modules_manager.instances
+        # Ok now all we need is the import module
+        self.modules_manager.set_modules([mig_mod])
+        self.do_load_modules()
+        print self.modules_manager.instances
+        if len(self.modules_manager.instances) == 0:
+            print "Error duringthe initialization of the import module. Bailing out"
+            sys.exit(2)
+        print "Configuration migrating in progress..."
+        mod  = self.modules_manager.instances[0]
+        f = getattr(mod, 'import_objects', None)
+        if not f or not callable(f):
+            print "Import module is missing the import_objects function. Bailing out"
+            sys.exit(2)
+
+        objs = {}
+        types = ['hosts', 'services', 'commands', 'timeperiods', 'contacts']
+        for t in types:
+            print "New type", t
+            objs[t] = []
+            for i in getattr(self.conf, t):
+                d = i.get_raw_import_values()
+                if d:
+                    objs[t].append(d)
+            f(objs)
+        # Ok we can exit now
+        sys.exit(0)
+
+
+
     # Main loop function
     def main(self):
         try:
@@ -486,6 +532,10 @@ def main(self):
 
             self.load_config_file()
 
+            # Maybe we are in a migration phase. If so, we will bailout here
+            if self.migrate:
+                self.go_migrate()
+
             # Look if we are enabled or not. If ok, start the daemon mode
             self.look_for_early_exit()
             self.do_daemon_init_and_start()

shinken/modules/mongodb_generic.py

@@ -29,6 +29,10 @@
 
 # This module imports hosts and services configuration from a MySQL Database
 # Queries for getting hosts and services are pulled from shinken-specific.cfg configuration file.
+try:
+    import uuid
+except ImportError:
+    uuid = None
 
 try:
     from pymongo.connection import Connection
@@ -70,21 +74,21 @@ def __init__(self, mod_conf, uri, database):
 
     # Called by Arbiter to say 'let's prepare yourself guy'
     def init(self):
-        print "[Mongodb Module]: Try to open a Mongodb connection to %s:%s" % (self.uri, self.database)
+        logger.info("[Mongodb Module]: Try to open a Mongodb connection to %s:%s" % (self.uri, self.database))
         try:
             self.con = Connection(self.uri)
             self.db = getattr(self.con, self.database)
         except Exception, e:
-            print "Mongodb Module: Error %s:" % e
+            logger.error("Mongodb Module: Error %s:" % e)
             raise
-        print "[Mongodb Module]: Connection OK"
+        logger.info("[Mongodb Module]: Connection OK")
 
 ################################ Arbiter part #################################
 
     # Main function that is called in the CONFIGURATION phase
     def get_objects(self):
         if not self.db:
-            print "[Mongodb Module]: Problem during init phase"
+            logger.error("[Mongodb Module]: Problem during init phase")
             return {}
 
         r = {}
@@ -95,7 +99,7 @@ def get_objects(self):
 
             cur = getattr(self.db, t).find({'_state': {'$ne': 'disabled'}})
             for h in cur:
-                print "DBG: mongodb: get an ", t, h
+                #print "DBG: mongodb: get an ", t, h
                 # We remove a mongodb specific property, the _id
                 del h['_id']
                 # And we add an imported_from property to say it came from
@@ -105,6 +109,59 @@ def get_objects(self):
 
         return r
 
+
+    def get_uniq_id(self, t, i):
+        #by default we will return a very uniq id
+        u = str(int(uuid.uuid4().int))
+
+        is_tpl = (getattr(i, 'register', '1')  == '0')
+        if is_tpl:
+            return 'tpl-%s' % getattr(i, 'name', u)
+
+        lst = {'hosts' : 'host_name', 'commands' : 'command_name',
+               'timeperiods' : 'timeperiod_name',
+               'contacts' : 'contact_name',
+               }
+        if t in lst:
+            prop = lst[t]
+            n = getattr(i, prop, None)
+            if n:
+                return n
+            return u
+        if t == 'services':
+            return u
+
+        print "Unknown TYPE in migration!"
+        return u
+
+
+
+    # Function called by the arbiter so we import the objects in our databases
+    def import_objects(self, data):
+        if not self.db:
+            logger.error("[Mongodb]: error Problem during init phase")
+            return False
+
+        # Maybe we can't have a good way to have uniq id, if so, bail out
+        if not uuid:
+            logger.error("Your python version is too old. Please update to a 2.6 version to use this feature")
+            return False
+
+
+        for t in data:
+            col = getattr(self.db, t)
+            print "Saving objects %s" % t
+            elts = data[t]
+            for e in elts:
+                print "Element", e
+                e['_id'] = self.get_uniq_id(t, e)
+                col.save(e)
+
+
+        return True
+
+
+
 #################################### WebUI parts ############################
 
     # We will get in the mongodb database the user preference entry, for the 'shinken-global' user

shinken/objects/item.py

@@ -369,6 +369,22 @@ def old_properties_names_to_new(self):
                 value = getattr(self, old_name)
                 setattr(self, new_name, value)
 
+    # The arbiter is asking us our raw value before all explode or linking
+    def get_raw_import_values(self):
+        r = {}
+        properties = self.__class__.properties.keys()
+        # Register is not by default in the properties
+        if not 'register' in properties:
+            properties.append('register')
+
+        for prop in properties:
+            if hasattr(self, prop):
+                v = getattr(self, prop)
+                print prop, ":", v
+                r[prop] = v
+        return r
+
+
     def add_downtime(self, downtime):
         self.downtimes.append(downtime)
 

shinken/objects/timeperiod.py

@@ -87,6 +87,7 @@
 
 import time
 import re
+import string
 
 from item import Item, Items
 
@@ -149,6 +150,23 @@ def get_unresolved_properties_by_inheritance(self, items):
         for i in self.templates:
             self.unresolved.extend(i.unresolved)
 
+    # Ok timeperiods are a bit different from classic items, because we do not have a real list
+    # of our raw rpoperties, like if we got february 1 - 15 / 3 for example
+    def get_raw_import_values(self):
+        properties = ['timeperiod_name', 'alias', 'use', 'register']
+        r = {}
+        for prop in properties:
+            if hasattr(self, prop):
+                v = getattr(self, prop)
+                print prop, ":", v
+                r[prop] = v
+        # Now the unresolved one. The only way to get ride of same key things is to put
+        # directly the full value as the key
+        for other in self.unresolved:
+            r[other] = ''
+        return r
+
+
     def is_time_valid(self, t):
         if self.has('exclude'):
             for dr in self.exclude:

shinken/webui/plugins_skonf/elements/elements.py

@@ -63,6 +63,7 @@ def elements_generic(cls, show_tpls=False):
     key = keys[t]
     #if cls.my_type == 'host':
     #    print "HOOK HOSTS"
+
     elts = [cls(i) for i in app.datamgr.get_generics(t, key)]
     print "GOT elements", elts
 

shinken/webui/plugins_skonf/elements/local_helper.py

@@ -35,7 +35,10 @@ def find(value, lst, key):
 
     for i in lst:
         v = i.get(key, None)
-        print 'MAtch with', v, value
+        try:
+            print 'MAtch with', v, value
+        except:
+            pass
         if v == value:
             return i
     return None
@@ -61,7 +64,10 @@ def find_several(lst, elt, prop, key):
         if v is None:
             continue
         v = v.strip()
-        print 'MAtch with db', v
+        try:
+            print 'MAtch with db', v
+        except:
+            pass
         if v  in values:
             res.append(dbi)
     print "Return find_several::", res