Merge pull request #37 from guilhemmarchand/version_1022

Version 1.0.22

build.sh

@@ -6,6 +6,9 @@ app="trackme"
 version=`grep 'version =' trackme/default/app.conf | awk '{print $3}' | sed 's/\.//g'`
 
 rm -f *.tgz
+find . -name "*.pyc" -type f | xargs rm -f
+find . -name "*.py" -type f | xargs chmod go-x
+find trackme/lib -name "*.py" -type f | xargs chmod a-x
 tar -czf ${app}_${version}.tgz --exclude=trackme/local --exclude=trackme/metadata/local.meta --exclude=trackme/lookups/lookup_file_backups trackme
 echo "Wrote: ${app}_${version}.tgz"
 

docs/configuration.rst

@@ -61,10 +61,24 @@ This is driven by the following macro definition:
 
 If you prefer to have a different format, customise this macro definition.
 
-Whitelisting of indexes
-=======================
+Whitelisting and blacklisting
+=============================
 
-The default behaviour of TrackMe is to track data available in all indexes, a macro is used within each tracker which can be modified if you wish to restrict TrackMe to a given list of indexes or any other valid Meta data filter:
+TrackMe version 1.0.22 introduced builtin support for both whitelisting of indexes and blacklisting of indexes, sourcetypes and hosts.
+
+.. image:: img/whitelist_and_blacklist.png
+   :alt: whitelist_and_blacklist.png
+   :align: center
+
+The default behaviour of TrackMe is to track data available in all indexes, which changes if whitelisting has been defined:
+
+.. image:: img/whitelisting.png
+   :alt: .png
+   :align: center
+
+The same UI allows you to define blacklisted items, which are added as part of a "NOT" boolean syntax within the searches.
+
+Finally, in addition the following macro is used within the searches, and can be customized if you need:
 
 ::
 
@@ -73,8 +87,8 @@ The default behaviour of TrackMe is to track data available in all indexes, a ma
     definition = sourcetype!="stash" sourcetype!="*too_small"
     iseval = 0
 
-Activation of out of the box alerts
-===================================
+Activation of builtin alerts
+============================
 
 **TrackMe provides out of the box alerts that be used to deliver alerting when a data source or host reaches a red alert:**
 

docs/releasenotes.rst

@@ -1,6 +1,21 @@
 Release notes
 #############
 
+Version 1.0.22
+==============
+
+- feature: Whitelisting major improvement with UI supported and driven whitelisting of indexes at data discovery and search time (Issue #27)
+- feature: Improve builtin choices for time input link selection within modal windows
+- feature: Abstract tracker saved searches to remove useless code redundancy
+- fix: Remove auto-refresh search link for searches which shouldn't be refreshed automatically (audit changes & flip, various collection management)
+- fix: Drilldown on any priority entities in alert should define monitored_state to enabled
+- fix: Monitor split share percentage error (Single forms shall share 25% each)
+- fix: Lagging class auditing can register an incorrect type of operation
+- fix: All time time range picker will not work for audit changes & status flipping
+- fix: Auto refresh set to none has random side effects on embedded chart loading, fixed by none set to long period
+- fix: Switched from default last 7 days to last 24 hours in audit and status flipping UIs
+- fix: TrackMe Mobile view does not honour blacklists
+
 Version 1.0.21
 ==============
 

docs/userguide.rst

@@ -266,8 +266,20 @@ When a lagging class is defined and is matched for a data source or a data host,
 
 Finally, when a custom lagging value is defined for an object, a value of "true" is created for the field named "data_override_lagging_class", which value is used to determine the actual value for that object.
 
-Blacklisting features
-=====================
+Whitelisting & Blacklisting features
+====================================
+
+**TrackMe version 1.0.22 introduced builtin support for both whitelisting of indexes and blacklisting of indexes, sourcetypes and hosts.**
+
+.. image:: img/whitelist_and_blacklist.png
+   :alt: whitelist_and_blacklist.png
+   :align: center
+
+**The default behaviour of TrackMe is to track data available in all indexes, which changes if whitelisting has been defined:**
+
+.. image:: img/whitelisting.png
+   :alt: .png
+   :align: center
 
 **Different level of blacklisting features are provided out of the box, which features can be used to avoid taking in consideration indexes, sourcetypes and hosts.**
 

trackme/bin/app.py

@@ -0,0 +1,114 @@
+# coding=utf-8
+#
+# Copyright © 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+""" Sets the packages path and optionally starts the Python remote debugging client.
+
+The Python remote debugging client depends on the settings of the variables defined in _pydebug_conf.py.  Set these
+variables in _pydebug_conf.py to enable/disable debugging using either the JetBrains PyCharm or Eclipse PyDev remote
+debug egg which must be copied to your application's bin directory and renamed as _pydebug.egg.
+
+"""
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+settrace = stoptrace = lambda: NotImplemented
+remote_debugging = None
+
+
+def initialize():
+
+    from os import path
+    from sys import modules, path as python_path
+
+    import platform
+
+    module_dir = path.dirname(path.realpath(__file__))
+    system = platform.system()
+
+    for packages in path.join(module_dir, 'packages'), path.join(path.join(module_dir, 'packages', system)):
+        if not path.isdir(packages):
+            break
+        python_path.insert(0, path.join(packages))
+
+    configuration_file = path.join(module_dir, '_pydebug_conf.py')
+
+    if not path.exists(configuration_file):
+        return
+
+    debug_client = path.join(module_dir, '_pydebug.egg')
+
+    if not path.exists(debug_client):
+        return
+
+    _remote_debugging = {
+        'client_package_location': debug_client,
+        'is_enabled': False,
+        'host': None,
+        'port': 5678,
+        'suspend': True,
+        'stderr_to_server': False,
+        'stdout_to_server': False,
+        'overwrite_prev_trace': False,
+        'patch_multiprocessing': False,
+        'trace_only_current_thread': False}
+
+    exec(compile(open(configuration_file).read(), configuration_file, 'exec'), {'__builtins__': __builtins__}, _remote_debugging)
+    python_path.insert(1, debug_client)
+
+    from splunklib.searchcommands import splunklib_logger as logger
+    import pydevd
+
+    def _settrace():
+        host, port = _remote_debugging['host'], _remote_debugging['port']
+        logger.debug('Connecting to Python debug server at %s:%d', host, port)
+
+        try:
+            pydevd.settrace(
+                host=host,
+                port=port,
+                suspend=_remote_debugging['suspend'],
+                stderrToServer=_remote_debugging['stderr_to_server'],
+                stdoutToServer=_remote_debugging['stdout_to_server'],
+                overwrite_prev_trace=_remote_debugging['overwrite_prev_trace'],
+                patch_multiprocessing=_remote_debugging['patch_multiprocessing'],
+                trace_only_current_thread=_remote_debugging['trace_only_current_thread'])
+        except SystemExit as error:
+            logger.error('Failed to connect to Python debug server at %s:%d: %s', host, port, error)
+        else:
+            logger.debug('Connected to Python debug server at %s:%d', host, port)
+
+    global remote_debugging
+    remote_debugging = _remote_debugging
+
+    global settrace
+    settrace = _settrace
+
+    global stoptrace
+    stoptrace = pydevd.stoptrace
+
+    remote_debugging_is_enabled = _remote_debugging['is_enabled']
+
+    if isinstance(remote_debugging_is_enabled, (list, set, tuple)):
+        app_name = path.splitext(path.basename(modules['__main__'].__file__))[0]
+        remote_debugging_is_enabled = app_name in remote_debugging_is_enabled
+
+    if remote_debugging_is_enabled is True:
+        settrace()
+
+    return
+
+initialize()
+del initialize

trackme/bin/getidxwhitelist.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# coding=utf-8
+#
+# Copyright © 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+import app
+import os,sys
+
+splunkhome = os.environ['SPLUNK_HOME']
+sys.path.append(os.path.join(splunkhome, 'etc', 'apps', 'searchcommands_app', 'lib'))
+from splunklib.searchcommands import dispatch, StreamingCommand, Configuration, Option, validators
+from splunklib import six
+
+
+@Configuration()
+class CountMatchesCommand(StreamingCommand):
+    """ Counts the number of non-overlapping matches to a regular expression in a set of fields.
+
+    ##Syntax
+
+    .. code-block::
+        countmatches fieldname=<field> pattern=<regular_expression> <field-list>
+
+    ##Description
+
+    A count of the number of non-overlapping matches to the regular expression specified by `pattern` is computed for
+    each record processed. The result is stored in the field specified by `fieldname`. If `fieldname` exists, its value
+    is replaced. If `fieldname` does not exist, it is created. Event records are otherwise passed through to the next
+    pipeline processor unmodified.
+
+    ##Example
+
+    Count the number of words in the `text` of each tweet in tweets.csv and store the result in `word_count`.
+
+    .. code-block::
+        | inputlookup tweets | countmatches fieldname=word_count pattern="\\w+" text
+
+    """
+    fieldname = Option(
+        doc='''
+        **Syntax:** **fieldname=***<fieldname>*
+        **Description:** Name of the field that will hold the match count''',
+        require=True, validate=validators.Fieldname())
+
+    pattern = Option(
+        doc='''
+        **Syntax:** **pattern=***<regular-expression>*
+        **Description:** Regular expression pattern to match''',
+        require=True, validate=validators.RegularExpression())
+
+    def stream(self, records):
+        self.logger.debug('CountMatchesCommand: %s', self)  # logs command line
+        pattern = self.pattern
+
+        count = 0
+        whitelist = ""
+
+        for record in records:
+
+            for fieldname in self.fieldnames:
+                matches = pattern.findall(six.text_type(record[fieldname].decode("utf-8")))
+                count += len(matches)
+            record[self.fieldname] = count
+
+            if whitelist != "":
+                whitelist = str(whitelist) + "|" + str(record)
+            else:
+                whitelist = str(record)
+
+        # whitelist is empty
+        if count == 0:
+            whitelist = "[('data_index', '*')]"
+
+        yield {'_raw': str(whitelist)}
+
+dispatch(CountMatchesCommand, sys.argv, sys.stdin, sys.stdout, __name__)

trackme/default/app.conf

@@ -16,4 +16,4 @@ label = TrackMe
 [launcher]
 author = Guilhem Marchand
 description = Easy data tracking system for Splunk admins
-version = 1.0.21
+version = 1.0.22

trackme/default/collections.conf

@@ -33,6 +33,20 @@ replicate = false
 [kv_trackme_data_source_monitoring_blacklist_sourcetype]
 replicate = false
 
+#
+# Indexes whitelisting
+#
+
+# whitelisting for data source monitoring
+
+[kv_trackme_data_source_monitoring_whitelist_index]
+replicate = false
+
+# whitelisting for data host monitoring
+
+[kv_trackme_data_host_monitoring_whitelist_index]
+replicate = false
+
 #
 # blacklisting for data host monitoring
 #

trackme/default/commands.conf

@@ -0,0 +1,7 @@
+# commands.conf
+
+# This custom command is used to evaluate the whitelist index input, specially if it is empty
+# as the default will, it needs to render a wildcard matching
+[getidxwhitelist]
+filename = getidxwhitelist.py
+chunked = true