Merge pull request #79 from earthgecko/luminosity

v1.2.7-stable-luminosity

docs/conf.py

@@ -103,7 +103,7 @@ def setup(app):
 # The short X.Y version.
 version = u'1.2'
 # The full version, including alpha/beta/rc tags.
-release = u'1.2.6-stable'
+release = u'1.2.7-stable'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/panorama.rst

@@ -13,7 +13,9 @@ this may result in a lot of entries in the anomalies DB table, it is useful for
 helping with root cause analysis.
 
 New records are used to trigger further analysis of the metric population in
-Luminosity.
+Luminosity to analyse the metric population for cross correlations.  However,
+only metrics with a :mod:`settings.ALERTS` tuple trigger Luminosity, but all
+metric anomalies are recorded.
 
 The :mod:`settings.PANORAMA_CHECK_MAX_AGE` ensures that Panorama only processes
 checks that are not older than this value.  This mitigates against Panorama
@@ -48,7 +50,16 @@ access the database with the user and password you configure in ``settings.py``
   really big data sets may invalidate this in the future, Gaia DR1 MySQL say :)
 
 - See ``skyline.sql`` in your cloned Skyline repo for the schema creation script
-- Enable Panorama and set the other Panorama settings in ``settings.py``
+  and create the skyline database (also create a user with permissions on the
+  database, e.g.
+
+```
+mysql -u root -p < /opt/skyline/github/skyline/skyline/skyline.sql
+mysql -u root -p -e "GRANT ALL ON skyline.* TO 'skyline'@'localhost' IDENTIFIED BY '$YOUR_MYSQL_SKYLINE_PASSWORD' \
+FLUSH PRIVILEGES;"
+```
+
+- Enable Panorama and review the other Panorama settings in ``settings.py``
 - Start Panorama (use your appropriate PATH) - or go back to `Installation`_ and
   continue with the installation steps and Panorama will be started later in the
   installation process.

docs/releases.rst

@@ -5,6 +5,7 @@ Release Notes
    :maxdepth: 1
    :glob:
 
+   releases/1_2_7
    releases/1_2_6
    releases/1_2_5
    releases/1_2_4

docs/releases/1_2_7.rst

@@ -0,0 +1,91 @@
+==============================
+1.2.7 - the luminosity branch
+==============================
+
+v1.2.7-luminosity - September 3, 2018
+
+Changes from v1.2.6
+-------------------
+
+- Bumped version to v1.2.7
+- Fix bug in alert on stale metrics #67 (2568)
+- Fix bug in seed_data.py #77 (2552)
+
+Update notes
+------------
+
+- NOTE: If you are running v1.x you CANNOT upgrade from v1.x directly to v1.2.7
+- There is no change to the DB or SQL
+- There are no changes to settings.py
+
+How to update from v1.2.2, v1.2.3, v1.2.4 or v1.2.5
+---------------------------------------------------
+
+- Download the new release tag or clone/update to get it to a temp location,
+  ready to be deployed.
+- Diff your current running version skyline/settings.py with the new version of
+  skyline/settings.py and create a new settings.py in the temp location
+  appropriate for Skyline v1.2.7 and your set up, ready to be deployed.
+- Stop all the Skyline services and ensure any service controls such as monit,
+  supervisor or systemd do not start them again, (change as appropriate for
+  your set up) e.g.
+
+.. code-block:: bash
+
+    NEW_SKYLINE_VERSION="v1.2.7-stable-luminosity"    # Your new Skyline version
+    OLD_SKYLINE_VERSION="v1.2.6-stable-luminosity"    # Your old Skyline version
+    # OR
+    # OLD_SKYLINE_VERSION="v1.2.2-stable-luminosity"
+    # OLD_SKYLINE_VERSION="v1.2.3-stable-luminosity"
+    # etc
+
+    CURRENT_SKYLINE_PATH="/opt/skyline/github/skyline"                 # Your Skyline path
+    NEW_SKYLINE_PATH="${CURRENT_SKYLINE_PATH}.${NEW_SKYLINE_VERSION}"  # Your new Skyline path
+
+    mkdir -p "${CURRENT_SKYLINE_PATH}.${NEW_SKYLINE_VERSION}"
+    cd "${CURRENT_SKYLINE_PATH}.${NEW_SKYLINE_VERSION}"
+    git clone https://github.com/earthgecko/skyline .
+    git checkout "$NEW_SKYLINE_VERSION"
+
+    # Stop all other Skyline services
+    SKYLINE_SERVICES="horizon
+    analyzer
+    mirage
+    crucible
+    boundary
+    ionosphere
+    luminosity
+    panorama
+    webapp"
+    for i in $SKYLINE_SERVICES
+    do
+      /etc/init.d/$i stop
+    done
+
+- Move your current Skyline directory to a backup directory and move the new
+  Skyline v1.2.7 with your new settings.py from the temp location to your
+  working Skyline directory, (change your paths as appropriate) e.g.
+
+.. code-block:: bash
+
+    mv /opt/skyline/github/skyline /opt/skyline/github/skyline.v1.2.6-luminosity-stable
+    mv "$NEW_SKYLINE_PATH" "$CURRENT_SKYLINE_PATH"
+
+- Start the all Skyline services (change as appropriate for your set up) e.g.
+
+.. code-block:: bash
+
+    # Start all other Skyline services
+    SKYLINE_SERVICES="panorama
+    luminosity
+    horizon
+    analyzer
+    mirage
+    crucible
+    boundary
+    ionosphere
+    webapp"
+    for i in $SKYLINE_SERVICES
+    do
+      /etc/init.d/$i start
+    done

skyline/analyzer/alerters.py

@@ -100,7 +100,7 @@ def alert_smtp(alert, metric, context):
 
     """
     LOCAL_DEBUG = False
-    logger = logging.getLogger(skyline_app_logger)
+    # logger = logging.getLogger(skyline_app_logger)
     if settings.ENABLE_DEBUG or LOCAL_DEBUG:
         logger.info('debug :: alert_smtp - sending smtp alert')
         logger.info('debug :: alert_smtp - Memory usage at start: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
@@ -959,48 +959,61 @@ def alert_stale_digest(alert, metric, context):
 
     """
     LOCAL_DEBUG = False
-    logger = logging.getLogger(skyline_app_logger)
     if settings.ENABLE_DEBUG or LOCAL_DEBUG:
         logger.info('debug :: alert_stale_digest - sending smtp digest alert of stale metrics')
 
+    # @added 20180828 - Bug #2568: alert on stale metrics not firing
+    # Create a new list from the metric[1] tuple item
+    logger.info('alert_stale_digest - metric :: %s' % str(metric))
+    stale_metrics = metric[1]
+    logger.info('alert_stale_digest - stale_metrics :: %s' % str(stale_metrics))
+
+    sender = settings.SMTP_OPTS['sender']
     try:
-        recipient = settings.SMTP_OPTS['default_recipient']
+        # @modified 20180828 - Bug #2568: alert on stale metrics not firing
+        # That was a list not a str
+        # recipient = settings.SMTP_OPTS['default_recipient']
+        recipient = settings.SMTP_OPTS['default_recipient'][0]
+        logger.info('alert_stale_digest - sending smtp digest alert to %s' % str(recipient))
     except:
         logger.error('error :: alert_stale_digest - no known default_recipient')
         return False
 
     try:
         body = '<h3><font color="#dd3023">Sky</font><font color="#6698FF">line</font><font color="black"> %s alert</font></h3><br>' % context
         body += '<font color="black"><b>Stale metrics (no data sent for ~%s seconds):</b></font><br>' % str(settings.ALERT_ON_STALE_PERIOD)
-        for metric_name in metric[1]:
+        # @modified 20180828 - Bug #2568: alert on stale metrics not firing
+        # for metric_name in stale_metric[1]:
+        for metric_name in stale_metrics:
             body += '<font color="black">%s</font><br>' % str(metric_name)
     except:
         logger.info(traceback.format_exc())
         logger.error('error :: alert_stale_digest - could not build body')
-        try:
-            msg = MIMEMultipart('mixed')
-            cs_ = charset.Charset('utf-8')
-            cs_.header_encoding = charset.QP
-            cs_.body_encoding = charset.QP
-            msg.set_charset(cs_)
-            msg['Subject'] = '[Skyline alert] - %s ALERT - stale metrics' % (context)
-            msg['From'] = sender
-            msg['To'] = recipient
-            msg.attach(MIMEText(body, 'html'))
-            msg.replace_header('content-transfer-encoding', 'quoted-printable')
-            msg.attach(MIMEText(more_body, 'html'))
-        except:
-            logger.error('error :: alert_smtp - could not attach')
-            logger.info(traceback.format_exc())
+    try:
+        msg = MIMEMultipart('mixed')
+        cs_ = charset.Charset('utf-8')
+        cs_.header_encoding = charset.QP
+        cs_.body_encoding = charset.QP
+        msg.set_charset(cs_)
+        msg['Subject'] = '[Skyline alert] - %s ALERT - %s stale metrics' % (context, str(len(stale_metrics)))
+        msg['From'] = sender
+        msg['To'] = recipient
+        msg.attach(MIMEText(body, 'html'))
+        msg.replace_header('content-transfer-encoding', 'quoted-printable')
+        #msg.attach(MIMEText(body, 'html'))
+        logger.info('alert_stale_digest - msg attached')
+    except:
+        logger.error('error :: alert_smtp - could not attach')
+        logger.info(traceback.format_exc())
 
-        s = SMTP('127.0.0.1')
-        try:
-            s.sendmail(sender, recipient, msg.as_string())
-            logger.info('alert_stale_digest - sent email to recipient :: %s' % str(recipient))
-        except:
-            logger.info(traceback.format_exc())
-            logger.error('error :: alert_stale_digest - could not send email to recipient :: %s' % str(recipient))
-        s.quit()
+    s = SMTP('127.0.0.1')
+    try:
+        s.sendmail(sender, recipient, msg.as_string())
+        logger.info('alert_stale_digest - sent email to recipient :: %s' % str(recipient))
+    except:
+        logger.info(traceback.format_exc())
+        logger.error('error :: alert_stale_digest - could not send email to recipient :: %s' % str(recipient))
+    s.quit()
     return
 
 

skyline/analyzer/algorithms.py

@@ -51,14 +51,14 @@
 # @added 20180807 - Feature #2492: alert on stale metrics
 try:
     from settings import ALERT_ON_STALE_METRICS
-    alert_on_stale_metrics = settings.ALERT_ON_STALE_METRICS
+#    ALERT_ON_STALE_METRICS = settings.ALERT_ON_STALE_METRICS
 except:
-    alert_on_stale_metrics = False
+    ALERT_ON_STALE_METRICS = False
 try:
     from settings import ALERT_ON_STALE_PERIOD
-    alert_on_stale_metrics = settings.ALERT_ON_STALE_PERIOD
+#    ALERT_ON_STALE_PERIOD = settings.ALERT_ON_STALE_PERIOD
 except:
-    alert_on_stale_metrics = 300
+    ALERT_ON_STALE_PERIOD = 300
 
 """
 This is no man's land. Do anything you want in here,
@@ -525,7 +525,7 @@ def run_selected_algorithm(timeseries, metric_name):
     # @added 20180807 - Feature #2492: alert on stale metrics
     # Determine if a metric has stopped sending data and if so add to the
     # analyzer.alert_on_stale_metrics Redis set
-    if alert_on_stale_metrics:
+    if ALERT_ON_STALE_METRICS:
         add_to_alert_on_stale_metrics = False
         # @modified 20180816 - Feature #2492: alert on stale metrics
         # Added try and except to prevent some errors that are encounter between
@@ -540,6 +540,8 @@ def run_selected_algorithm(timeseries, metric_name):
             if int(time()) - int(timeseries[-1][0]) >= ALERT_ON_STALE_PERIOD:
                 add_to_alert_on_stale_metrics = True
         except:
+            # @modified 20180816 -
+            #                      Feature #2492: alert on stale metrics
             add_to_alert_on_stale_metrics = False
         try:
             if int(time()) - int(timeseries[-1][0]) >= STALE_PERIOD:

skyline/analyzer/analyzer.py

@@ -1690,8 +1690,10 @@ def smtp_trigger_alert(alert, metric, context):
             # @added 20180807 - Feature #2492: alert on stale metrics
             if settings.ALERT_ON_STALE_METRICS:
                 stale_metrics_to_alert_on = []
+                alert_on_stale_metrics = []
                 try:
                     alert_on_stale_metrics = list(self.redis_conn.smembers('analyzer.alert_on_stale_metrics'))
+                    logger.info('alert_on_stale_metrics :: %s' % str(alert_on_stale_metrics))
                 except:
                     alert_on_stale_metrics = []
                 for metric_name in alert_on_stale_metrics:
@@ -1719,7 +1721,10 @@ def smtp_trigger_alert(alert, metric, context):
                     if not last_alert:
                         stale_metrics_to_alert_on.append(base_name)
                         try:
-                            self.redis_conn.setex(cache_key, int(time()), int(settings.STALE_PERIOD))
+                            # @modified 20180828 - Bug #2568: alert on stale metrics not firing
+                            # Those expired in the year 2067
+                            # self.redis_conn.setex(cache_key, int(time()), int(settings.STALE_PERIOD))
+                            self.redis_conn.setex(cache_key, int(settings.STALE_PERIOD), int(time()))
                         except:
                             logger.error('error :: failed to add %s Redis key' % cache_key)
                 if stale_metrics_to_alert_on:
@@ -1738,6 +1743,7 @@ def smtp_trigger_alert(alert, metric, context):
                     del metric
                 else:
                     logger.info('there are no stale metrics to alert on')
+                del stale_metrics_to_alert_on
 
             run_time = time() - now
             total_metrics = str(len(unique_metrics))

skyline/skyline_version.py

@@ -4,7 +4,7 @@
 # @modified 20170109 - Feature #1854: Ionosphere learn
 # Added learn
 # __version_info__ = ('1', '1', '0')
-__version_info__ = ('1', '2', '6')
+__version_info__ = ('1', '2', '7')
 __branch__ = 'luminosity'
 __version_tag__ = 'stable'