Merge pull request #1106 from mozilla/add_index_open_and_close_abstra…

…ction

Adding open and close abstraction for managing indexes

CHANGELOG

@@ -60,7 +60,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ### Fixed
 - Resolved sshd mq plugin to handle more types of events
 
-
 ## [v1.37] - 2019-03-01
 ### Added
 - Watchlist - use the UI to quickly add a term (username, IP, command, etc.) that MozDef alerts on

cron/closeIndices.conf

@@ -0,0 +1,3 @@
+[options]
+esservers = http://localhost:9200
+index_age = 30

cron/closeIndices.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2014 Mozilla Corporation
+
+# set this to run as a cronjob (after backup has completed)
+# to regularly remove indexes
+
+# .conf file will determine what indexes are operated on
+# Create a starter .conf file with backupDiscover.py
+
+import sys
+from datetime import datetime, timedelta
+from configlib import getConfig, OptionParser
+
+from mozdef_util.utilities.logger import logger
+from mozdef_util.utilities.toUTC import toUTC
+from mozdef_util.elasticsearch_client import ElasticsearchClient
+
+
+def esCloseIndices():
+    logger.debug('started')
+    try:
+        es = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers)))
+        indices = es.get_indices()
+    except Exception as e:
+        logger.error("Unhandled exception while connecting to ES, terminating: %r" % (e))
+
+    # examine each index pulled from get_indice
+    # to determine if it meets aging criteria
+    month_ago_date = toUTC(datetime.now()) - timedelta(days=int(options.index_age))
+    month_ago_date = month_ago_date.replace(tzinfo=None)
+    for index in indices:
+        if 'events' in index:
+            index_date = index.rsplit('-', 1)[1]
+            logger.debug("Checking to see if Index: %s can be closed." % (index))
+            if len(index_date) == 8:
+                index_date_obj = datetime.strptime(index_date, '%Y%m%d')
+                try:
+                    if month_ago_date > index_date_obj:
+                        logger.debug("Index: %s will be closed." % (index))
+                        es.close_index(index)
+                    else:
+                        logger.debug("Index: %s  does not meet aging criteria and will not be closed." % (index))
+                except Exception as e:
+                    logger.error("Unhandled exception while closing indices, terminating: %r" % (e))
+
+
+def initConfig():
+    # output our log to stdout or syslog
+    options.output = getConfig(
+        'output',
+        'stdout',
+        options.configfile
+    )
+    # syslog hostname
+    options.sysloghostname = getConfig(
+        'sysloghostname',
+        'localhost',
+        options.configfile
+    )
+    options.syslogport = getConfig(
+        'syslogport',
+        514,
+        options.configfile
+    )
+    options.esservers = list(getConfig(
+        'esservers',
+        'http://localhost:9200',
+        options.configfile).split(',')
+    )
+    options.index_age = getConfig(
+        'index_age',
+        15,
+        options.configfile
+    )
+
+
+if __name__ == '__main__':
+    parser = OptionParser()
+    parser.add_option("-c",
+                      dest='configfile',
+                      default=sys.argv[0].replace('.py', '.conf'),
+                      help="configuration file to use")
+    (options, args) = parser.parse_args()
+    initConfig()
+    esCloseIndices()

cron/closeIndices.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2014 Mozilla Corporation
+
+source  /opt/mozdef/envs/python/bin/activate
+/opt/mozdef/envs/mozdef/cron/closeIndices.py -c /opt/mozdef/envs/mozdef/cron/closeIndices.conf

mozdef_util/HISTORY.rst

@@ -41,3 +41,9 @@ Add is_ip utility function
 ------------------
 
 * Add get_aliases function to elasticsearch client
+
+
+1.0.7 (2019-04-07)
+------------------
+
+* Add close_index and open_index functions to elasticsearch client

mozdef_util/mozdef_util/elasticsearch_client.py

@@ -38,11 +38,16 @@ def __init__(self, servers, bulk_amount=100, bulk_refresh_time=30):
         self.es_connection.ping()
         self.bulk_queue = BulkQueue(self, threshold=bulk_amount, flush_time=bulk_refresh_time)
 
+    def close_index(self, index_name):
+        return self.es_connection.indices.close(index=index_name)
+
+    def open_index(self, index_name):
+        return self.es_connection.indices.open(index=index_name)
+
     def delete_index(self, index_name, ignore_fail=False):
         ignore_codes = []
         if ignore_fail is True:
             ignore_codes = [400, 404]
-
         self.es_connection.indices.delete(index=index_name, ignore=ignore_codes)
 
     def get_indices(self):

mozdef_util/setup.py

@@ -56,6 +56,6 @@
     test_suite='tests',
     tests_require=[],
     url='https://github.com/mozilla/MozDef/tree/master/lib',
-    version='1.0.6',
+    version='1.0.7',
     zip_safe=False,
 )

requirements.txt

@@ -31,7 +31,7 @@ jmespath==0.9.3
 kombu==4.1.0
 meld3==1.0.2
 mozdef-client==1.0.11
-mozdef-util==1.0.6
+mozdef-util==1.0.7
 MySQL-python==1.2.5
 netaddr==0.7.1
 nose==1.3.7

tests/mozdef_util/test_elasticsearch_client.py

@@ -120,6 +120,54 @@ def test_search_no_results(self):
         assert results['hits'] == []
 
 
+class TestCloseIndex(ElasticsearchClientTest):
+
+    def teardown(self):
+        super(TestCloseIndex, self).teardown()
+        if pytest.config.option.delete_indexes:
+            self.es_client.delete_index('test_index')
+
+    def test_close_index(self):
+        if not self.es_client.index_exists('test_index'):
+            self.es_client.create_index('test_index')
+        time.sleep(1)
+        closed = self.es_client.close_index('test_index')
+        assert closed == {'acknowledged': True}
+
+
+class TestWritingToClosedIndex(ElasticsearchClientTest):
+
+    def teardown(self):
+        super(TestWritingToClosedIndex, self).teardown()
+        if pytest.config.option.delete_indexes:
+            self.es_client.delete_index('test_index')
+
+    def test_writing_to_closed_index(self):
+        if not self.es_client.index_exists('test_index'):
+            self.es_client.create_index('test_index')
+        time.sleep(1)
+        self.es_client.close_index('test_index')
+        event = json.dumps({"key": "example value for string of json test"})
+        with pytest.raises(Exception):
+            self.es_client.save_event(index='test_index', body=event)
+
+
+class TestOpenIndex(ElasticsearchClientTest):
+
+    def teardown(self):
+        super(TestOpenIndex, self).teardown()
+        if pytest.config.option.delete_indexes:
+            self.es_client.delete_index('test_index')
+
+    def test_index_open(self):
+        if not self.es_client.index_exists('test_index'):
+            self.es_client.create_index('test_index')
+        time.sleep(1)
+        self.es_client.close_index('test_index')
+        opened = self.es_client.open_index('test_index')
+        assert opened == {'acknowledged': True}
+
+
 class TestWithBadIndex(ElasticsearchClientTest):
 
     def test_search_nonexisting_index(self):