Merge pull request #243 from guilhemmarchand/version_1234

Version 1.2.34

.gitignore

@@ -40,3 +40,6 @@ appinspect_report*.html
 # backup directory
 trackme/backup
 
+# build directory
+build/*.tgz
+build/release-sha256.txt

README.md

@@ -4,6 +4,12 @@
 | ---    | ---          |
 | master | [![master status](https://circleci.com/gh/guilhemmarchand/trackme/tree/master.svg?style=svg)](https://circleci.com/gh/guilhemmarchand/trackme/tree/master)
 
+## Download
+
+**Consult the application releases in Git:**
+
+https://github.com/guilhemmarchand/trackme/releases
+
 ## The Splunk TrackMe application provides automated monitoring and visibility insight of your data sources availability, with a powerful user interface and workflow for Splunk product owners to detect and alert on failures or abnormal latency:
 
 - Discover and store key states information of data sources, data hosts and metric hosts availability

build/build.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+#set -x
+
+# for Mac OS X
+export COPYFILE_DISABLE=true
+
+PWD=`pwd`
+app="trackme"
+cp -a ../${app} .
+version=`grep 'version =' ${app}/default/app.conf | awk '{print $3}' | sed 's/\.//g'`
+
+rm -f *.tgz
+find . -name "*.pyc" -type f | xargs rm -f
+find . -name "*.py" -type f | xargs chmod go-x
+find trackme/lib -name "*.py" -type f | xargs chmod a-x
+tar -czf ${app}_${version}.tgz --exclude=${app}/local --exclude=${app}/backup --exclude=${app}/metadata/local.meta --exclude=${app}/lookups/lookup_file_backups --exclude=${app}/default.old* --exclude='./.*'  --exclude='.[^/]*' --exclude="._*" ${app}
+
+sha256=$(sha256sum ${app}_${version}.tgz)
+echo "Wrote: ${sha256}"
+echo ${sha256} > release-sha256.txt
+
+rm -rf ${app}
+exit 0

docs/releasenotes.rst

@@ -1,6 +1,23 @@
 Release notes
 #############
 
+Version 1.2.34
+==============
+
+**CAUTION:**
+
+This is a new main release branch, TrackMe 1.2.x requires the deployment of the following dependencies:
+
+- Semicircle Donut Chart Viz, Splunk Base: https://splunkbase.splunk.com/app/4378
+- Splunk Machine Learning Toolkit, Splunk Base: https://splunkbase.splunk.com/app/2890
+- Splunk Timeline - Custom Visualization, Splunk Base: https://splunkbase.splunk.com/app/3120
+
+TrackMe requires a summary index (defaults to trackme_summary) and a metric index (defaults to trackme_metrics):
+https://trackme.readthedocs.io/en/latest/configuration.html
+
+- Enhancement - Issue #241 - KVstore backup and restore - Improved workflow with Metadata recording of backup archives, new dashboard providing insights on the workflow and its features
+- Fix - Issues #242 - UI - interfaces like lagging classes, allow and block listing should not remove the search input form if there are no results found
+
 Version 1.2.33
 ==============
 

docs/userguide.rst

@@ -3066,16 +3066,30 @@ Each archive contains a JSON file corresponding to the entire content of the KVs
 
 To perform a restore operation (see the documentation following), the relevant tarball archive needs to be located in the same directory.
 
+When a backup is taken, a record with Metadata is added in a dedicated KVstore collection (kv_trackme_backup_archives_info), records are automatically purged when the archive is deleted due to retention. (any missing archive record is as well added if discovered on a search head when a get backups command runs)
+
 For Splunk Cloud certification purposes, the application will never attempt to write or access a directory ouf of the application name space level.
 
 .. admonition:: notes about Search Head Clustering (SHC)
 
    - If TrackMe is deployed in a Search Head Cluster, the scheduled report is executed on a single search head, randomly
    - As such, the archive file is created on this specific instance, but not replicated to other members
-   - Restoring requires to manually locate the server hosting the archive file, and running the restore command from this node especially
+   - Restoring requires to locate the server hosting the archive file using the audit dashboard or manually in the Metadata collection, and running the restore command from this node especially
    - The restore operation does not mandatory requires to be executed from the SHC / KVstore captain
    - in a SHC context, the purging part of schedule report happens only on the member running the report, therefore archive files can exist longer than the retention on other members
 
+Backup and Restore dashboard
+----------------------------
+
+**An auditing dashboard is provided in the app navigation menu "API & Tooling" that provides an overview of the backup archives knowledge and statuses:**
+
+.. image:: img/backup_and_restore/dashboard_backup_and_restore.png
+   :alt: dashboard_backup_and_restore.png
+   :align: center
+   :width: 1200px
+
+This dashboard uses the backup archives Metadata stores in the KVstore collection **trackme_backup_archives_info** to show the list of backups that were taken over time per instance.
+
 Automatic backup
 ----------------
 

trackme/app.manifest

@@ -5,7 +5,7 @@
     "id": {
       "group": null,
       "name": "trackme",
-      "version": "1.2.33"
+      "version": "1.2.34"
     },
     "author": [
       {

trackme/bin/trackme_rest_handler_backup_and_restore.py

@@ -2,11 +2,13 @@
 import os, sys, shutil
 import tarfile
 import csv
-import time
+import time, datetime
 import splunk
 import splunk.entity
 import splunk.Intersplunk
 import json
+import socket
+import hashlib
 
 logger = logging.getLogger(__name__)
 
@@ -25,7 +27,7 @@ class TrackMeHandlerBackupAndRestore_v1(rest_handler.RESTHandler):
     def __init__(self, command_line, command_arg):
         super(TrackMeHandlerBackupAndRestore_v1, self).__init__(command_line, command_arg, logger)
 
-    # List backup archive files on this instance
+    # List backup archive files known from the KV, add any local file that the KV wouldn't do about
     def get_backup(self, request_info, **kwargs):
 
         describe = False
@@ -67,6 +69,9 @@ def get_backup(self, request_info, **kwargs):
             # Set backup root dir
             backuproot = os.path.join(splunkhome, 'etc', 'apps', 'trackme', 'backup')
 
+            # get local server name
+            server_name = socket.gethostname()
+
             # check backup dir existence
             if not os.path.isdir(backuproot):
 
@@ -82,11 +87,76 @@ def get_backup(self, request_info, **kwargs):
                 from os.path import isfile, join
                 backup_files = [join(backuproot, f) for f in listdir(backuproot) if isfile(join(backuproot, f))]
 
-                return {
-                    "payload": "{\"backup_files\": \"" + str(backup_files) + "\"}",
-                    'status': 200 # HTTP status code
-                }
+                if not backup_files:
+
+                    return {
+                        "payload": "{\"There are no backup archives available on this instance\"}",
+                        'status': 200 # HTTP status code
+                    }
+
+                else:
+
+                    # Enter the list, verify for that for each archive file we have a corresponding record in the audit collection
+                    # if there is no record, then we create a replacement record
+                    for backup_file in backup_files:
+
+                        # get the file mtime
+                        backup_file_mtime = round(os.path.getmtime(backup_file))
+
+                        # Store a record in a backup audit collection
+
+                        # Create a message
+                        status_message = "discovered"
+
+                        # record / key
+                        record = None
+                        key = None
+
+                        # Define the KV query
+                        query_string = '{ "$and": [ { "backup_archive": "' + backup_file + '" }, { "server_name' + '": "' + server_name + '" } ] }'
+
+                        # backup audit collection
+                        collection_name_backup_archives_info = "kv_trackme_backup_archives_info"            
+                        service_backup_archives_info = client.connect(
+                            owner="nobody",
+                            app="trackme",
+                            port=splunkd_port,
+                            token=request_info.session_key
+                        )
+                        collection_backup_archives_info = service_backup_archives_info.kvstore[collection_name_backup_archives_info]
+
+                        try:
+                            record = collection_backup_archives_info.data.query(query=str(query_string))
+                            key = record[0].get('_key')
+
+                        except Exception as e:
+                            key = None
 
+                        # If a record cannot be found, this backup file is not know to TrackMe currently, add a new record
+                        if key is None:
+
+                            try:
+
+                                # Insert the record
+                                collection_backup_archives_info.data.insert(json.dumps({
+                                    "mtime": str(backup_file_mtime),
+                                    "htime": str(datetime.datetime.fromtimestamp(int(backup_file_mtime)).strftime('%c')),
+                                    "server_name": str(server_name),
+                                    "status": str(status_message),
+                                    "change_type": "backup archive was missing from the info collection and added by automatic discovery",
+                                    "backup_archive": str(backup_file)
+                                    }))
+
+                            except Exception as e:
+                                return {
+                                    'payload': 'Warn: exception encountered: ' + str(e) # Payload of the request.
+                                }
+
+                    # Render
+                    return {
+                        "payload": collection_backup_archives_info.data.query(),
+                        'status': 200 # HTTP status code
+                    }
 
     # Take a backup
     def post_backup(self, request_info, **kwargs):
@@ -136,6 +206,9 @@ def post_backup(self, request_info, **kwargs):
             # Set backup dir
             backupdir = os.path.join(backuproot, str(timestr))
 
+            # get local server name
+            server_name = socket.gethostname()
+
             # Create the backup dir if does not exist
             if not os.path.isdir(backuproot):
                 os.mkdir(backuproot)
@@ -233,10 +306,45 @@ def post_backup(self, request_info, **kwargs):
                     'payload': "Error: %s : %s" % (backupdir, e.strerror)
                 }
 
-            # Render
+            # Store a record in a backup audit collection
+
+            # backup audit collection
+            collection_name_backup_archives_info = "kv_trackme_backup_archives_info"            
+            service_backup_archives_info = client.connect(
+                owner="nobody",
+                app="trackme",
+                port=splunkd_port,
+                token=request_info.session_key
+            )
+            collection_backup_archives_info = service_backup_archives_info.kvstore[collection_name_backup_archives_info]
+
+            # Get time
+            current_time = int(round(time.time()))
+
+            # Create a message
+            status_message = "{\"backup_archive\": \"" + str(tar_name) + "\", \"report\": \""\
+                    + str(counter_performed) + " collections backed up / " + str(counter_empty) + " collections empty\"}"
+
+            try:
+
+                # Insert the record
+                collection_backup_archives_info.data.insert(json.dumps({
+                    "mtime": str(current_time),
+                    "htime": str(datetime.datetime.fromtimestamp(int(current_time)).strftime('%c')),
+                    "server_name": str(server_name),
+                    "status": str(status_message),
+                    "change_type": "backup archive created",
+                    "backup_archive": str(os.path.join(backupdir, tar_name))
+                    }))
+
+            except Exception as e:
+                return {
+                    'payload': 'Warn: exception encountered: ' + str(e) # Payload of the request.
+                }
+
+            # Finally render a status message
             return {
-                "payload": "{ \"backup_archive\": \"" + str(tar_name) + "\", \"report\": \""\
-                    + str(counter_performed) + " collections backed up / " + str(counter_empty) + " collections empty\"}",
+                "payload": str(status_message),
                 'status': 200 # HTTP status code
             }
 
@@ -247,6 +355,14 @@ def delete_backup(self, request_info, **kwargs):
         describe = False
         retention_days = 7 # default to 7 days of retention if not specified
 
+        # Get splunkd port
+        entity = splunk.entity.getEntity('/server', 'settings',
+                                            namespace='trackme', sessionKey=request_info.session_key, owner='-')
+        splunkd_port = entity['mgmtHostPort']
+
+        # get local server name
+        server_name = socket.gethostname()
+
         # Retrieve from data
         try:
             resp_dict = json.loads(str(request_info.raw_args['payload']))
@@ -344,6 +460,39 @@ def delete_backup(self, request_info, **kwargs):
                 }
             else:
 
+                # For each backup archive, if a record is found in the collection info, remove the record
+
+                # backup audit collection
+                collection_name_backup_archives_info = "kv_trackme_backup_archives_info"            
+                service_backup_archives_info = client.connect(
+                    owner="nobody",
+                    app="trackme",
+                    port=splunkd_port,
+                    token=request_info.session_key
+                )
+                collection_backup_archives_info = service_backup_archives_info.kvstore[collection_name_backup_archives_info]
+
+                for backup_file in purgedlist:
+
+                    key = None
+                    record = None
+
+                    # Define the KV query
+                    query_string = '{ "$and": [ { "backup_archive": "' + backup_file + '" }, { "server_name' + '": "' + server_name + '" } ] }'
+
+                    try:
+                        record = collection_backup_archives_info.data.query(query=str(query_string))
+                        key = record[0].get('_key')
+
+                    except Exception as e:
+                        key = None
+
+                    # If a record is found, it shall be purged
+                    if key is not None:
+
+                        # Remove the record
+                        collection_backup_archives_info.data.delete(json.dumps({"_key":key}))
+
                 return {
                     "payload": "{\"status\": \"The following archive files were purged due to retention (" + str(retention_days)\
                     + " days)\", \"backup_files\": \"" + str(purgedlist) + "\"}",

trackme/default/app.conf

@@ -16,4 +16,4 @@ label = TrackMe
 [launcher]
 author = Guilhem Marchand
 description = Data tracking system for Splunk
-version = 1.2.33
+version = 1.2.34

trackme/default/collections.conf

@@ -200,3 +200,9 @@ replicate = false
 #
 [kv_trackme_tags_policies]
 replicate = false
+
+#
+# Backup collection, used to store server / backup files Metadata
+#
+[kv_trackme_backup_archives_info]
+replicate = false