Skip to content
Permalink
Browse files

Merge pull request #1106 from mozilla/add_index_open_and_close_abstra…

…ction

Adding open and close abstraction for managing indexes
  • Loading branch information...
Phrozyn committed Apr 10, 2019
2 parents 8024171 + 8dd29af commit 5cefe7c9afa254eb584d686a1d0d4802192ef3b5
@@ -60,7 +60,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
### Fixed
- Resolved sshd mq plugin to handle more types of events


## [v1.37] - 2019-03-01
### Added
- Watchlist - use the UI to quickly add a term (username, IP, command, etc.) that MozDef alerts on
@@ -0,0 +1,3 @@
[options]
esservers = http://localhost:9200
index_age = 30
@@ -0,0 +1,89 @@
#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2014 Mozilla Corporation

# set this to run as a cronjob (after backup has completed)
# to regularly remove indexes

# .conf file will determine what indexes are operated on
# Create a starter .conf file with backupDiscover.py

import sys
from datetime import datetime, timedelta
from configlib import getConfig, OptionParser

from mozdef_util.utilities.logger import logger
from mozdef_util.utilities.toUTC import toUTC
from mozdef_util.elasticsearch_client import ElasticsearchClient


def esCloseIndices():
logger.debug('started')
try:
es = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers)))
indices = es.get_indices()
except Exception as e:
logger.error("Unhandled exception while connecting to ES, terminating: %r" % (e))

# examine each index pulled from get_indice
# to determine if it meets aging criteria
month_ago_date = toUTC(datetime.now()) - timedelta(days=int(options.index_age))
month_ago_date = month_ago_date.replace(tzinfo=None)
for index in indices:
if 'events' in index:
index_date = index.rsplit('-', 1)[1]
logger.debug("Checking to see if Index: %s can be closed." % (index))
if len(index_date) == 8:
index_date_obj = datetime.strptime(index_date, '%Y%m%d')
try:
if month_ago_date > index_date_obj:
logger.debug("Index: %s will be closed." % (index))
es.close_index(index)
else:
logger.debug("Index: %s does not meet aging criteria and will not be closed." % (index))
except Exception as e:
logger.error("Unhandled exception while closing indices, terminating: %r" % (e))


def initConfig():
# output our log to stdout or syslog
options.output = getConfig(
'output',
'stdout',
options.configfile
)
# syslog hostname
options.sysloghostname = getConfig(
'sysloghostname',
'localhost',
options.configfile
)
options.syslogport = getConfig(
'syslogport',
514,
options.configfile
)
options.esservers = list(getConfig(
'esservers',
'http://localhost:9200',
options.configfile).split(',')
)
options.index_age = getConfig(
'index_age',
15,
options.configfile
)


if __name__ == '__main__':
parser = OptionParser()
parser.add_option("-c",
dest='configfile',
default=sys.argv[0].replace('.py', '.conf'),
help="configuration file to use")
(options, args) = parser.parse_args()
initConfig()
esCloseIndices()
@@ -0,0 +1,9 @@
#!/usr/bin/env bash

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2014 Mozilla Corporation

source /opt/mozdef/envs/python/bin/activate
/opt/mozdef/envs/mozdef/cron/closeIndices.py -c /opt/mozdef/envs/mozdef/cron/closeIndices.conf
@@ -41,3 +41,9 @@ Add is_ip utility function
------------------

* Add get_aliases function to elasticsearch client


1.0.7 (2019-04-07)
------------------

* Add close_index and open_index functions to elasticsearch client
@@ -38,11 +38,16 @@ def __init__(self, servers, bulk_amount=100, bulk_refresh_time=30):
self.es_connection.ping()
self.bulk_queue = BulkQueue(self, threshold=bulk_amount, flush_time=bulk_refresh_time)

def close_index(self, index_name):
return self.es_connection.indices.close(index=index_name)

def open_index(self, index_name):
return self.es_connection.indices.open(index=index_name)

def delete_index(self, index_name, ignore_fail=False):
ignore_codes = []
if ignore_fail is True:
ignore_codes = [400, 404]

self.es_connection.indices.delete(index=index_name, ignore=ignore_codes)

def get_indices(self):
@@ -56,6 +56,6 @@
test_suite='tests',
tests_require=[],
url='https://github.com/mozilla/MozDef/tree/master/lib',
version='1.0.6',
version='1.0.7',
zip_safe=False,
)
@@ -31,7 +31,7 @@ jmespath==0.9.3
kombu==4.1.0
meld3==1.0.2
mozdef-client==1.0.11
mozdef-util==1.0.6
mozdef-util==1.0.7
MySQL-python==1.2.5
netaddr==0.7.1
nose==1.3.7
@@ -120,6 +120,54 @@ def test_search_no_results(self):
assert results['hits'] == []


class TestCloseIndex(ElasticsearchClientTest):

def teardown(self):
super(TestCloseIndex, self).teardown()
if pytest.config.option.delete_indexes:
self.es_client.delete_index('test_index')

def test_close_index(self):
if not self.es_client.index_exists('test_index'):
self.es_client.create_index('test_index')
time.sleep(1)
closed = self.es_client.close_index('test_index')
assert closed == {'acknowledged': True}


class TestWritingToClosedIndex(ElasticsearchClientTest):

def teardown(self):
super(TestWritingToClosedIndex, self).teardown()
if pytest.config.option.delete_indexes:
self.es_client.delete_index('test_index')

def test_writing_to_closed_index(self):
if not self.es_client.index_exists('test_index'):
self.es_client.create_index('test_index')
time.sleep(1)
self.es_client.close_index('test_index')
event = json.dumps({"key": "example value for string of json test"})
with pytest.raises(Exception):
self.es_client.save_event(index='test_index', body=event)


class TestOpenIndex(ElasticsearchClientTest):

def teardown(self):
super(TestOpenIndex, self).teardown()
if pytest.config.option.delete_indexes:
self.es_client.delete_index('test_index')

def test_index_open(self):
if not self.es_client.index_exists('test_index'):
self.es_client.create_index('test_index')
time.sleep(1)
self.es_client.close_index('test_index')
opened = self.es_client.open_index('test_index')
assert opened == {'acknowledged': True}


class TestWithBadIndex(ElasticsearchClientTest):

def test_search_nonexisting_index(self):

0 comments on commit 5cefe7c

Please sign in to comment.
You can’t perform that action at this time.