Merge pull request #105 from earthgecko/luminosity

slack

dev-requirements.txt

@@ -300,7 +300,9 @@ python-simple-hipchat==0.4.0
 #pygerduty==0.36.2
 # @modified 20190412 - Task #2926: Update dependencies
 #pygerduty==0.37.0
-pygerduty==0.38.1
+# @modified 20190508 - Task #2926: Update dependencies
+#pygerduty==0.38.1
+pygerduty==0.38.2
 
 # for panorama - direct from MySQL as the package has broken in the context of
 # pip and virtualenv - NOTE: once you have installed this once, if it is run in

docs/conf.py

@@ -103,7 +103,7 @@ def setup(app):
 # The short X.Y version.
 version = u'1.2'
 # The full version, including alpha/beta/rc tags.
-release = u'1.2.15'
+release = u'1.2.16'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/readthedocs.requirements.txt

@@ -2,6 +2,7 @@
 #                      Branch #2270: luminosity
 # @modified 20190412 - Task #2926: Update dependencies
 # @modified 20190426 - Task #2964: Update dependencies
+# @modified 20190508 - Task #2926: Update dependencies
 # Use pip 10.0.1 as may be a readthedocs issue - testing
 pip>=10.0.1
 setuptools
@@ -39,7 +40,7 @@ statsmodels==0.9.0
 msgpack-python==0.5.6
 requests==2.21.0
 python-simple-hipchat==0.4.0
-pygerduty==0.38.1
+pygerduty==0.38.2
 mysql-connector-python==8.0.16
 gunicorn==19.9.0
 pipdeptree==0.13.2

docs/releases.rst

@@ -5,6 +5,7 @@ Release Notes
    :maxdepth: 1
    :glob:
 
+   releases/1_2_16
    releases/1_2_15
    releases/1_2_14
    releases/1_2_13

docs/releases/1_2_16.rst

@@ -0,0 +1,169 @@
+======
+1.2.16
+======
+
+v1.2.16 - May 4, 2019
+
+This is a features and dependencies update release.
+
+Changes in v1.2.16
+------------------
+
+- Dependencies updated (2964)
+- Report defaulted settings to log.  Added all the globally declared settings to
+  enable reporting in the log the state of each setting (2950)
+- Added Skyline slack thread updates on alerts (2646)
+
+Update notes
+------------
+
+- These update instruction apply to upgrading from v1.2.15 to v1.2.16 only.
+  However as with all Skyline updates it is possible to go through the update
+  notes for each version and make your own update notes/process to take you from
+  version x to version y.
+- There are changes to the DB in v1.2.16
+- There are changes to settings.py in v1.2.16
+
+How to update from v1.2.15
+--------------------------
+
+- Download the new release tag or clone/update to get it to a temp location,
+  ready to be deployed.
+- Diff your current running version skyline/settings.py with the new version of
+  skyline/settings.py and create a new settings.py in the temp location
+  appropriate for Skyline v1.2.16 and your instance with the new
+  :mod:`settings.SLACK_OPTS` dictionary options, ready to be deployed.  The new
+  options that have been added to the `SLACK_OPTS` dictionary are shown below.
+  Replace the `YOUR_` variables with your channel and channel_id strings.  You
+  can set the message_on options for each event type.  If you do not want
+  Skyline to update slack threads, set `thread_updates: False`:
+
+.. code-block:: python
+
+      # Your default slack Skyline channel name e.g. '#skyline'
+      'default_channel': 'YOUR_default_slack_channel',
+      # Your default slack Skyline channel id e.g. 'C0XXXXXX'
+      'default_channel_id': 'YOUR_default_slack_channel_id',
+      # Whether to update slack message threads on any of the below events
+      'thread_updates': True,
+      # You can disable or enable each message_on event
+      'message_on_training_data_viewed': True,
+      'message_on_training_data_viewed_reaction_emoji': 'eyes',
+      'message_on_features_profile_created': True,
+      'message_on_features_profile_created_reaction_emoji': 'thumbsup',
+      'message_on_features_profile_learnt': True,
+      'message_on_features_profile_learnt_reaction_emoji': 'heavy_check_mark',
+      'message_on_features_profile_disabled': True,
+      'message_on_features_profile_disabled_reaction_emoji': 'x',
+      'message_on_validated_features_profiles': True,
+
+
+- Stop Skyline Ionosphere and Panorama services.
+- Backup your Skyline database.
+- Apply the SQL update to your Skyline database as per described below, change
+  the below variables as appropriate to your set up.
+- Stop all the Skyline services and ensure any service controls such as monit,
+  supervisor or systemd do not start them again, (change as appropriate for
+  your set up)
+
+.. code-block:: bash
+
+    NEW_SKYLINE_VERSION="v1.2.16"    # Your new Skyline version
+    OLD_SKYLINE_VERSION="v1.2.15"    # Your old Skyline version
+
+    CURRENT_SKYLINE_PATH="/opt/skyline/github/skyline"                 # Your Skyline path
+    NEW_SKYLINE_PATH="${CURRENT_SKYLINE_PATH}.${NEW_SKYLINE_VERSION}"  # Your new Skyline path
+
+    mkdir -p "${CURRENT_SKYLINE_PATH}.${NEW_SKYLINE_VERSION}"
+    cd "${CURRENT_SKYLINE_PATH}.${NEW_SKYLINE_VERSION}"
+    git clone https://github.com/earthgecko/skyline .
+    git checkout "$NEW_SKYLINE_VERSION"
+
+    cp "$NEW_SKYLINE_PATH/skyline/settings.py" "$NEW_SKYLINE_PATH/skyline/settings.py.${NEW_SKYLINE_VERSION}.bak"
+    cat "${CURRENT_SKYLINE_PATH}/skyline/settings.py" > "$NEW_SKYLINE_PATH/skyline/settings.py"
+
+    # ADD the new SLACK_OPTS to settings
+    vi "$NEW_SKYLINE_PATH/skyline/settings.py"
+
+    # Stop Skyline DB related services
+    SKYLINE_SERVICES="ionosphere
+    luminosity
+    panorama"
+    for i in $SKYLINE_SERVICES
+    do
+      /etc/init.d/$i stop
+    done
+    # And stop any service controls like monit or systemd
+
+    # BACKUP THE DB AND APPLY THE NEW SQL
+    BACKUP_DIR="/tmp"  # Where you want to backup the DB to
+    MYSQL_USER="<YOUR_MYSQL_USER>"
+    MYSQL_HOST="127.0.0.1"  # Your MySQL IP
+    MYSQL_DB="skyline"  # Your MySQL Skyline DB name
+
+    # Backup DB
+    mkdir -p $BACKUP_DIR
+    mysqldump -u$MYSQL_USER -p $MYSQL_DB > $BACKUP_DIR/pre.$NEW_SKYLINE_VERSION.$MYSQL_DB.sql
+
+    # Update DB
+    mysql -u$MYSQL_USER -p $MYSQL_DB < "${NEW_SKYLINE_PATH}/updates/sql/v1.2.16-stable.sql"
+
+    # Stop all other Skyline services
+    SKYLINE_SERVICES="horizon
+    analyzer
+    mirage
+    crucible
+    boundary
+    ionosphere
+    luminosity
+    panorama
+    webapp"
+    for i in $SKYLINE_SERVICES
+    do
+      /etc/init.d/$i stop
+    done
+
+- Move your current Skyline directory to a backup directory and move the new
+  Skyline v1.2.16 with your new settings.py from the temp location to your
+  working Skyline directory, (change your paths as appropriate) e.g.
+
+.. code-block:: bash
+
+    mv "$CURRENT_SKYLINE_PATH" "${CURRENT_SKYLINE_PATH}.${OLD_SKYLINE_VERSION}"
+    mv "$NEW_SKYLINE_PATH" "$CURRENT_SKYLINE_PATH"
+
+- Start the all Skyline services (change as appropriate for your set up) e.g.
+
+.. code-block:: bash
+
+    # Start all other Skyline services
+    SKYLINE_SERVICES="panorama
+    luminosity
+    horizon
+    analyzer
+    mirage
+    crucible
+    boundary
+    ionosphere
+    webapp"
+    for i in $SKYLINE_SERVICES
+    do
+      /etc/init.d/$i start
+    done
+
+- Check the logs
+
+.. code-block:: bash
+
+    # How are they running
+    tail -n 20 /var/log/skyline/*.log
+
+    # Any errors - each app
+    find /var/log/skyline -type f -name "*.log" | while read skyline_logfile
+    do
+      echo "#####
+    # Checking for errors in $skyline_logfile"
+      cat "$skyline_logfile" | grep -B2 -A10 -i "error ::\|traceback" | tail -n 60
+      echo ""
+      echo ""
+    done

docs/running-multiple-skylines.rst

@@ -26,12 +26,75 @@ correlation of all the metrics in the population.  Skyline does this via a api
 endpoint and preprocesses the time series data for Luminosity on the remote
 Skyline instance and returns only the fragments (gzipped) of time series
 required for analysis, by default the previous 12 minutes, to minimise bandwidth
-and ensure maintain performance.
+and ensure performance is maintained.
+
+Hot standby configuration
+-------------------------
+
+Although there are many possible methods and configurations to ensure that
+single points of failure are mitigated in infrastructure, this can be difficult
+to achieve with both Graphite and Skyline.  This is due to the nature and volume
+of the data being dealt with, especially if you are interested in ensuring
+that you have redundant storage for disaster recovery.
+
+With Graphite it is difficult to ensure the whisper data is redundant, due to
+volume and real time nature of the whisper data files.
+
+With Skyline it is difficult to ensure the real time Redis data is redundant
+given that you DO NOT want to run a Redis slave as the ENTIRE key store
+constantly changes.  Slaving a Skyline Redis instance is not an option
+as it will use mountains of network bandwidth and just would not work.
+
+One possible configuration to achieve redundancy of Graphite and Skyline data is
+to run a Graphite and a Skyline instance as hot standbys in a different data
+center.  Where the primary Graphite is pickling to a primary Skyline instance
+and a standby Graphite instance.  With the standby Graphite instance pickling
+data to the standby Skyline instance.
+
+.. code-block
+
+                        graphite-1
+                            |
+                       carbon-relay
+          __________________|____
+          |            |         |
+      carbon-cache   pickle    pickle
+                       |         |
+                       |         +-->--> data-center-2
+                       |                      |
+                   skyline-1              graphite-2
+                                              |
+                                         carbon-relay
+                                        ______|______
+                                        |            |
+                                  carbon-cache     pickle
+                                                     |
+                                                  skyline-2
+
+In terms of the Skyline configuration of the hot standby you configure skyline-2
+the same as skyline-1 in terms of alerts, etc, but you set
+:mod:`settings.ANALYZER_ENABLED` and :mod:`settings.LUMINOSITY_ENABLED` to
+`False`.
+
+In the event of a failure of graphite-1 you reconfigure your things to send
+their metrics to graphite-2 and set skyline-2 :mod:`settings.ANALYZER_ENABLED`
+and :mod:`settings.LUMINOSITY_ENABLED` to `True`.
+
+In the event of a failure of skyline-1 you set skyline-2
+:mod:`settings.ANALYZER_ENABLED` and :mod:`settings.LUMINOSITY_ENABLED` to
+`True`.
+
+The setting up of a hot standby Graphite instance requires pickling AND periodic
+flock rsyncing of all the whisper files from graphite-1 to graphite-2 to ensure
+that any data that graphite-2 may have been lost in any `fullQueueDrops`
+experienced with the pickle from graphite-1 to graphite-2 due to network
+partitioning, etc, are updated.  flock rsyncing all the whisper files daily
+mostly handles this and ensures that you have no gaps in the whisper data on
+your backup Graphite instance.
 
 Webapp UI
 ---------
 
-Please note that not all the functionality in the Webapp UI is compatible or
-multiple instance aware.  However where any "not in Redis" UI errors are found,
-accessing the Skyine Webapp UI on which the metric resides in Redis will resolve
-the issue.
+In terms of the functionality in webapp, the webapp is multiple instance aware.
+Where any "not in Redis" UI errors are found, webapp responds to the request
+with a 301 redirect to the alternate Skyline URL.

docs/slack.rst

@@ -0,0 +1,67 @@
+slack
+=====
+
+Skyline can be integrated with slack in terms of alerts and making automatic
+updates to slack alert threads, informing you that a training data page has been
+reviewed, a features profile has been created or learnt and validated.
+
+To use slack with Skyline you need to create a slack skyline app and a bot user.
+
+On slack:
+Create a Slack App
+Basic Information > Building Apps for Slack > Add features and functionality > Bots
+Bot User click Add a Bot User
+Set Display name and Default username to skyline
+And Save Changes
+In the left hand Settings menu for the skyline app, click Basic Information
+In App Credentials click Add features and functionality > Permissions
+Under OAuth Tokens & Redirect URLs click Install App to Workspace and click Authorize
+Copy the Bot User OAuth Access Token and add it to :mod:`settings.SLACK_OPTS` as the value for bot_user_oauth_access_token
+If you want to add the Skyline icon to the app, in the left hand Settings menu for the skyline app, click Basic Information
+Under Display Information > App icon & Preview and click and add:
+https://github.com/earthgecko/skyline/blob/master/docs/images/slack.skyline.app.icon.png
+
+Create a new channel called skyline and invite the skyline bot user.
+
+Skyline slack settings
+----------------------
+
+In `settings.py` set
+
+.. code-block:: python
+
+  SLACK_ENABLED = True
+
+And update the the `SLACK_OPTS` as appropriate for your metrics.
+
+.. code-block:: python
+
+  SLACK_OPTS = {
+      # Bot User OAuth Access Token
+      'bot_user_oauth_access_token': 'YOUR_slack_bot_user_oauth_access_token',
+      # list of slack channels to notify about each anomaly
+      # (similar to SMTP_OPTS['recipients'])
+      # channel names - you can either pass the channel name (#general) or encoded
+      # ID (C024BE91L)
+      'channels': {
+          'skyline': ('#skyline',),
+          'skyline_test.alerters.test': ('#skyline',),
+          'horizon.udp.test': ('#skyline',),
+      },
+      'icon_emoji': ':chart_with_upwards_trend:',
+      # Your default slack Skyline channel name e.g. '#skyline'
+      'default_channel': 'YOUR_default_slack_channel',
+      # Your default slack Skyline channel id e.g. 'C0XXXXXX'
+      'default_channel_id': 'YOUR_default_slack_channel_id',
+      # Whether to update slack message threads on any of the below events
+      'thread_updates': True,
+      'message_on_training_data_viewed': True,
+      'message_on_training_data_viewed_reaction_emoji': 'eyes',
+      'message_on_features_profile_created': True,
+      'message_on_features_profile_created_reaction_emoji': 'thumbsup',
+      'message_on_features_profile_learnt': True,
+      'message_on_features_profile_learnt_reaction_emoji': 'heavy_check_mark',
+      'message_on_features_profile_disabled': True,
+      'message_on_features_profile_disabled_reaction_emoji': 'x',
+      'message_on_validated_features_profiles': True,
+  }

requirements.txt

@@ -33,7 +33,7 @@ statsmodels==0.9.0
 msgpack-python==0.5.6
 requests==2.21.0
 python-simple-hipchat==0.4.0
-pygerduty==0.38.1
+pygerduty==0.38.2
 mysql-connector-python==8.0.16
 gunicorn==19.9.0
 pipdeptree==0.13.2