diff --git a/benchmarking/es/insert_bulk.js b/benchmarking/es/insert_bulk.js deleted file mode 100644 index f4ac64c83..000000000 --- a/benchmarking/es/insert_bulk.js +++ /dev/null @@ -1,79 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. -// Copyright (c) 2014 Mozilla Corporation - -// Usage: node ./insert_bulk.js [host2] [host3] [...] - -var cluster = require('cluster'); -var http = require('http'); -var processes; -var insertsPerQuery; -var totalInserts; -var hosts = []; - -processes = parseInt(process.argv[2]); -insertsPerQuery = parseInt(process.argv[3]); -totalInserts = parseInt(process.argv[4]); - -if (process.argv.length < 6) { - console.error("Usage: node ./insert_bulk.js [host2] [host3] [...]"); - process.exit(1); -} - -process.argv.forEach(function(val, index, array) { - if (index > 4) { - hosts.push(val); - } -}); - -if (cluster.isMaster) { - // Fork workers. - for (var k = 0; k < processes; k++) { - cluster.fork(); - } - - cluster.on('exit', function(worker, code, signal) { - console.log('worker ' + worker.process.pid + ' died'); - }); -} else { - - var i = 0; - var body; - - while(42) { - var host = hosts[(i % hosts.length)]; - var body = ''; - - for(var j=0; j [host2] [host3] [...] - -var processes; -var totalInserts; -var hosts = []; - -var cluster = require('cluster'); -var http = require('http'); -var i = 0; - -processes = parseInt(process.argv[2]); -totalInserts = parseInt(process.argv[3]); - -if (process.argv.length < 5) { - console.error("Usage: node ./insert_simple.js [host2] [host3] [...]"); - process.exit(1); -} - -process.argv.forEach(function(val, index, array) { - if (index > 3) { - hosts.push(val); - } -}); - -if (cluster.isMaster) { - // Fork workers. - for (var k = 0; k < processes; k++) { - cluster.fork(); - } - - cluster.on('exit', function(worker, code, signal) { - console.log('worker ' + worker.process.pid + ' died'); - }); -} else { - - while(42) { - var host = hosts[(i % hosts.length)]; - var body = { - "user" : "kimchy"+i, - "post_date" : "2009-11-15T14:12:12", - "message" : "trying out Elasticsear "+i+" ch" - }; - - http.request({ - host: host, - port: 9200, - path: '/test1/tweet/'+i, - method: 'PUT' - }, function(res) { - res.on('data', function (chunk) { - //console.log('BODY: ' + chunk); - console.log('done '+i); - i -= 1; - }); - } - ).on('error', function(e) { - console.log("Got error: " + e.message); - }).end(JSON.stringify(body)) - - i += 1; - console.log(i); - if (i === totalInserts) { - break; - } - } -} - diff --git a/benchmarking/es/search_all_fulltext.js b/benchmarking/es/search_all_fulltext.js deleted file mode 100644 index 6d75f6a84..000000000 --- a/benchmarking/es/search_all_fulltext.js +++ /dev/null @@ -1,66 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. -// Copyright (c) 2014 Mozilla Corporation - -// Usage: node ./search_all_fulltext.js [host2] [host3] [...] - -var processes; -var totalSearches; -var hosts = []; - -var cluster = require('cluster'); -var http = require('http'); -var i = 0; - -processes = parseInt(process.argv[2]); -totalSearches = parseInt(process.argv[3]); - -if (process.argv.length < 5) { - console.error("Usage: node ./search_all_fulltext.js [host2] [host3] [...]"); - process.exit(1); -} - -process.argv.forEach(function(val, index, array) { - if (index > 3) { - hosts.push(val); - } -}); - -if (cluster.isMaster) { - // Fork workers. - for (var k = 0; k < processes; k++) { - cluster.fork(); - } - - cluster.on('exit', function(worker, code, signal) { - console.log('worker ' + worker.process.pid + ' died'); - }); -} else { - - while(42) { - var host = hosts[(i % hosts.length)]; - - http.request({ - host: host, - port: 9200, - path: '_search?q='+i, - method: 'GET' - }, function(res) { - res.on('data', function (chunk) { - //console.log('BODY: ' + chunk); - console.log('done '+i); - i -= 1; - }); - } - ).on('error', function(e) { - console.log("Got error: " + e.message); - }).end() - - i += 1; - console.log(i); - if (i === totalSearches) { - break; - } - } -} diff --git a/benchmarking/workers/json2Mozdef.py b/benchmarking/workers/json2Mozdef.py deleted file mode 100755 index 66d181c63..000000000 --- a/benchmarking/workers/json2Mozdef.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. -# Copyright (c) 2014 Mozilla Corporation - -import os -import sys -from datetime import datetime -import pytz -import json -import socket -from optparse import OptionParser -from requests_futures.sessions import FuturesSession -from multiprocessing import Process, Queue -import logging -from queue import Empty -from requests.packages.urllib3.exceptions import ClosedPoolError -import time - -httpsession = FuturesSession(max_workers=5) -httpsession.trust_env = False # turns of needless .netrc check for creds -# a = requests.adapters.HTTPAdapter(max_retries=2) -# httpsession.mount('http://', a) - - -logger = logging.getLogger(sys.argv[0]) -logger.level = logging.DEBUG - -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - - -def postLogs(logcache): - # post logs asynchronously with requests workers and check on the results - # expects a queue object from the multiprocessing library - posts = [] - try: - while not logcache.empty(): - postdata = logcache.get_nowait() - if len(postdata) > 0: - url = options.url - a = httpsession.get_adapter(url) - a.max_retries = 3 - r = httpsession.post(url,data=postdata) - posts.append((r,postdata,url)) - except Empty as e: - pass - for p, postdata, url in posts: - try: - if p.result().status_code >= 500: - logger.error("exception posting to %s %r [will retry]\n" % (url, p.result().status_code)) - # try again later when the next message in forces other attempts at posting. - logcache.put(postdata) - except ClosedPoolError as e: - # logger.fatal("Closed Pool Error exception posting to %s %r %r [will retry]\n"%(url,e,postdata)) - logcache.put(postdata) - except Exception as e: - logger.fatal("exception posting to %s %r %r [will not retry]\n" % (url, e, postdata)) - sys.exit(1) - - -if __name__ == '__main__': - parser=OptionParser() - parser.add_option("-u", dest='url', default='http://localhost:8080/events/', help="mozdef events URL to use when posting events") - (options, args) = parser.parse_args() - sh = logging.StreamHandler(sys.stdout) - sh.setFormatter(formatter) - logger.addHandler(sh) - # create a list of logs we can append json to and call for a post when we want. - logcache = Queue() - try: - for i in range(0,10): - - print(i) - alog = dict( - eventtime=pytz.timezone('UTC').localize(datetime.now()).isoformat(), - hostname=socket.gethostname(), - processid=os.getpid(), - processname=sys.argv[0], - severity='INFO', - summary='joe login failed', - category='authentication', - tags=[], - details=[] - ) - alog['details'] = dict(success=True, username='mozdef') - alog['tags'] = ['mozdef', 'stresstest'] - - logcache.put(json.dumps(alog)) - if not logcache.empty(): - time.sleep(.001) - try: - postingProcess = Process(target=postLogs, args=(logcache,), name="json2MozdefStressTest") - postingProcess.start() - except OSError as e: - if e.errno == 35: # resource temporarily unavailable. - print(e) - pass - else: - logger.error('%r' % e) - - while not logcache.empty(): - try: - postingProcess = Process(target=postLogs, args=(logcache,), name = "json2MozdefStressTest") - postingProcess.start() - except OSError as e: - if e.errno == 35: # resource temporarily unavailable. - print(e) - pass - else: - logger.error('%r' % e) - except KeyboardInterrupt as e: - sys.exit(1) diff --git a/docker/compose/dev-sampledata.yml b/docker/compose/dev-sampledata.yml index 2f3a2d9d9..6bae9fc44 100644 --- a/docker/compose/dev-sampledata.yml +++ b/docker/compose/dev-sampledata.yml @@ -6,7 +6,7 @@ services: context: ../../ dockerfile: docker/compose/mozdef_sampledata/Dockerfile restart: always - command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && /opt/mozdef/envs/mozdef/examples/demo/sampleevents.sh' + command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && /opt/mozdef/envs/mozdef/scripts/demo/populate_sample_events.sh' links: - elasticsearch depends_on: @@ -17,4 +17,4 @@ services: networks: - default volumes: - - ../../examples:/opt/mozdef/envs/mozdef/examples \ No newline at end of file + - ../../scripts/demo:/opt/mozdef/envs/mozdef/scripts/demo \ No newline at end of file diff --git a/docker/compose/mozdef_sampledata/Dockerfile b/docker/compose/mozdef_sampledata/Dockerfile index 2aff281bb..2d49629c4 100644 --- a/docker/compose/mozdef_sampledata/Dockerfile +++ b/docker/compose/mozdef_sampledata/Dockerfile @@ -2,11 +2,6 @@ FROM mozdef/mozdef_base LABEL maintainer="mozdef@mozilla.com" -COPY --chown=mozdef:mozdef ./examples /opt/mozdef/envs/mozdef/examples +COPY --chown=mozdef:mozdef scripts/demo /opt/mozdef/envs/mozdef/scripts/demo -COPY --chown=mozdef:mozdef docker/compose/mozdef_sampledata/files/sampleData2MozDef.conf /opt/mozdef/envs/mozdef/examples/demo/sampleData2MozDef.conf -RUN chmod u+rwx /opt/mozdef/envs/mozdef/examples/demo/sampleevents.sh - -WORKDIR /opt/mozdef/envs/mozdef/examples/demo - -USER root +WORKDIR /opt/mozdef/envs/mozdef/scripts/demo diff --git a/docker/compose/mozdef_sampledata/files/sampleData2MozDef.conf b/docker/compose/mozdef_sampledata/files/sampleData2MozDef.conf deleted file mode 100644 index e809301e3..000000000 --- a/docker/compose/mozdef_sampledata/files/sampleData2MozDef.conf +++ /dev/null @@ -1,4 +0,0 @@ -[options] -alertsminutesinterval = 10 -attackersminutesinterval = 10 -url = http://nginx:8080/events/ \ No newline at end of file diff --git a/docker/compose/tester/Dockerfile b/docker/compose/tester/Dockerfile index 3f39d2eb3..f28ddc161 100644 --- a/docker/compose/tester/Dockerfile +++ b/docker/compose/tester/Dockerfile @@ -6,7 +6,7 @@ RUN pip install -r /opt/mozdef/envs/mozdef/tests/requirements_tests.txt COPY alerts /opt/mozdef/envs/mozdef/alerts COPY bot /opt/mozdef/envs/mozdef/bot COPY cron /opt/mozdef/envs/mozdef/cron -COPY examples /opt/mozdef/envs/mozdef/examples +COPY scripts /opt/mozdef/envs/mozdef/scripts COPY loginput /opt/mozdef/envs/mozdef/loginput COPY mozdef_util /opt/mozdef/envs/mozdef/mozdef_util COPY mq /opt/mozdef/envs/mozdef/mq diff --git a/docs/source/benchmarking.rst b/docs/source/benchmarking.rst deleted file mode 100644 index 639dbae05..000000000 --- a/docs/source/benchmarking.rst +++ /dev/null @@ -1,59 +0,0 @@ -Benchmarking -============ - -Performance is important for a SIEM because it's where you want to store, search and analyze all your security events. - -You will want it to handle a significant number of new events per second, be able to search quickly and perform fast correlation. -Therefore, we provide some benchmarking scripts for MozDef to help you determine the performance of your setup. Performance tuning of elastic search can be complex and we highly recommend spending time tuning your environment. - - -Elasticsearch -------------- - -Elasticsearch is the main backend component of MozDef. -We strongly recommend you to have a 3+ nodes cluster to allow recovery and load balancing. -During our tests, Elasticsearch recovered well after being pushed to the limits of hardware, loosing and regaining nodes, and a variety of valid/invalid data. We provide the following scripts for you to use to test your own implementation. - -The scripts for Elasticsearch benchmarking are in `benchmarking/es/`. -They use `nodejs`_ to allow asynchronous HTTP requests. - -.. _nodejs: https://nodejs.org/ - -insert_simple.js -**************** - -`insert_simple.js` sends indexing requests with 1 log/request. - -Usage: `node ./insert_simple.js [host2] [host3] [...]` - - * `processes`: Number of processes to spawn - * `totalInserts`: Number of inserts to perform - * Please note after a certain number node will slow down. You want to have a lower number if you are in this case. - * `host1`, `host2`, `host3`, etc: Elasticsearch hosts to which you want to send the HTTP requests - -insert_bulk.js -************** - -`insert_bulk.js` sends bulk indexing requests (several logs/request). - -Usage: `node ./insert_bulk.js [host2] [host3] [...]` - - * `processes`: Number of processes to spawn - * `insertsPerQuery`: Number of logs per request - * `totalInserts`: Number of inserts to perform - * Please note after a certain number node will slow down. You want to have a lower number if you are in this case. - * `host1`, `host2`, `host3`, etc: Elasticsearch hosts to which you want to send the HTTP requests - -search_all_fulltext.js -********************** - -`search_all_fulltext.js` performs search on all indices, all fields in fulltext. It's very stupid. - -Usage: `node ./search_all_fulltext.js [host2] [host3] [...]` - - * `processes`: Number of processes to spawn - * `totalSearches`: Number of search requests to perform - * Please note after a certain number node will slow down. You want to have a lower number if you are in this case. - * `host1`, `host2`, `host3`, etc: Elasticsearch hosts to which you want to send the HTTP requests - - diff --git a/docs/source/index.rst b/docs/source/index.rst index 4ab0591bd..35ad815b7 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -20,7 +20,6 @@ Table of Contents cloud_deployment advanced_settings code - benchmarking cicd contributors diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 96f9dfcb1..b6f3eed68 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -539,12 +539,11 @@ To initialize elasticsearch indices and create sample kibana dashboards:: To add some sample data:: - (mozdef) [mozdef@mozdev mozdef]$ cd ~/mozdef/examples/demo - (mozdef) [mozdef@mozdev es-docs]$ python inject.py + (mozdef) [mozdef@mozdev mozdef]$ cd ~/mozdef/scripts/demo + (mozdef) [mozdef@mozdev demo]$ python populate_sample_events.py --num-rounds=1 -To add more sample data you can run the following from inside the venv:: +To continously add sample data:: - (mozdef) [mozdef@mozdev mozdef]$ cd ~/mozdef/examples/demo - (mozdef) [mozdef@mozdev demo]$ ./syncalerts.sh - (mozdef) [mozdef@mozdev demo]$ ./sampleevents.sh + (mozdef) [mozdef@mozdev mozdef]$ cd ~/mozdef/scripts/demo + (mozdef) [mozdef@mozdev demo]$ python populate_sample_events.py diff --git a/examples/demo/sampleData2MozDef.conf b/examples/demo/sampleData2MozDef.conf deleted file mode 100644 index d5e814403..000000000 --- a/examples/demo/sampleData2MozDef.conf +++ /dev/null @@ -1,4 +0,0 @@ -[options] -alertsminutesinterval = 10 -attackersminutesinterval = 10 - diff --git a/examples/demo/sampleData2MozDef.py b/examples/demo/sampleData2MozDef.py deleted file mode 100755 index 30d605d6b..000000000 --- a/examples/demo/sampleData2MozDef.py +++ /dev/null @@ -1,344 +0,0 @@ -#!/usr/bin/env python - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. -# Copyright (c) 2014 Mozilla Corporation - -import os -import sys -from datetime import datetime -import pytz -import json -from multiprocessing import Process, Queue -import random -import logging -from queue import Empty -import requests -import time -from configlib import getConfig, OptionParser -import configparser -import glob -from datetime import timedelta - -from mozdef_util.utilities.toUTC import toUTC - -# use futures to run in the background -# httpsession = FuturesSession(max_workers=5) -httpsession = requests.session() -httpsession.trust_env=False # turns of needless .netrc check for creds -# a = requests.adapters.HTTPAdapter(max_retries=2) -# httpsession.mount('http://', a) - - -logger = logging.getLogger(sys.argv[0]) -logger.level=logging.INFO - -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - -# create a list of logs we can append json to and call for a post when we want. -logcache=Queue() - - -def setConfig(option,value,configfile): - """write an option/value pair to our config file""" - if os.path.isfile(configfile): - config = configparser.ConfigParser() - configfp=open(configfile,'r') - config.readfp(configfp) - configfp.close() - - config.set('options',option,value) - configfp=open(configfile,'w') - config.write(configfp) - configfp.close() - - -def postLogs(logcache): - # post logs asynchronously with requests workers and check on the results - # expects a queue object from the multiprocessing library - # posts=[] - try: - while not logcache.empty(): - postdata=logcache.get_nowait() - if len(postdata)>0: - url=options.url - a=httpsession.get_adapter(url) - a.max_retries=3 - r=httpsession.post(url,data=postdata) - print(r) - # append to posts if this is long running and you want - # events to try again later. - # posts.append((r,postdata,url)) - except Empty: - pass - # for p, postdata, url in posts: - # try: - # if p.result().status_code >= 500: - # logger.error("exception posting to %s %r [will retry]\n" % (url, p.result().status_code)) - # # try again later when the next message in forces other attempts at posting. - # logcache.put(postdata) - # except ClosedPoolError as e: - # logger.fatal("Closed Pool Error exception posting to %s %r %r [will retry]\n" % (url, e, postdata)) - # logcache.put(postdata) - # except Exception as e: - # logger.fatal("exception posting to %s %r %r [will not retry]\n" % (url, e, postdata)) - # sys.exit(1) - - -def genRandomIPv4(): - # random, IPs - return '.'.join("%d" % (random.randint(0,254)) for x in range(4)) - - -def genAttackerIPv4(): - # random, but not too random as to allow for alerting about attacks from - # the same IP. - coreIPs=['1.93.25.', - '222.73.115.', - '116.10.191.', - '144.0.0.'] - # change this to non zero according to taste for semi-random-ness - if random.randint(0,10)>= 0: - return '{0}{1}'.format(random.choice(coreIPs), random.randint(1,2)) - else: - return '.'.join("%d" % (random.randint(0,254)) for x in range(4)) - - -def makeEvents(): - try: - eventfiles = glob.glob(options.eventsglob) - # pick a random number of events to send - for i in range(1, random.randrange(20, 100)): - # pick a random type of event to send - eventfile = random.choice(eventfiles) - # print(eventfile) - events = json.load(open(eventfile)) - target = random.randint(0, len(events)) - for event in events[target:target + 1]: - event['timestamp'] = pytz.timezone('UTC').localize(datetime.utcnow()).isoformat() - # remove stored times - if 'utctimestamp' in event: - del event['utctimestamp'] - if 'receivedtimestamp' in event: - del event['receivedtimestamp'] - - # add demo to the tags so it's clear it's not real data. - if 'tags' not in event: - event['tags'] = list() - - event['tags'].append('demodata') - - # replace potential with a random ip address - if 'summary' in event and '' in event['summary']: - randomIP = genRandomIPv4() - event['summary'] = event['summary'].replace("", randomIP) - if 'details' not in event: - event['details'] = dict() - event['details']['sourceipaddress'] = randomIP - event['details']['sourceipv4address'] = randomIP - - # print(event['timestamp'], event['tags'], event['summary']) - - logcache.put(json.dumps(event)) - if not logcache.empty(): - time.sleep(.01) - try: - postingProcess = Process(target=postLogs, args=(logcache,), name="json2MozdefDemoData") - postingProcess.start() - except OSError as e: - if e.errno == 35: # resource temporarily unavailable. - print(e) - pass - else: - logger.error('%r' % e) - - except KeyboardInterrupt as e: - sys.exit(1) - - -def makeAlerts(): - ''' - send events that will be correlated into alerts - ''' - try: - # time for us to run? - timetoRun = toUTC(options.lastalert) + timedelta(minutes=options.alertsminutesinterval) - if timetoRun > toUTC(datetime.now()): - # print(timetoRun) - return - - # print(timetoRun, options.lastalert) - eventfiles = glob.glob(options.alertsglob) - # pick a random number of events to send - for i in range(0, options.alertscount): - # pick a random type of event to send - eventfile = random.choice(eventfiles) - events = json.load(open(eventfile)) - target = random.randint(0, len(events)) - # if there's only one event in the file..use it. - if len(events) == 1 and target == 1: - target = 0 - for event in events[target:target + 1]: - event['timestamp'] = pytz.timezone('UTC').localize(datetime.utcnow()).isoformat() - # remove stored times - if 'utctimestamp' in event: - del event['utctimestamp'] - if 'receivedtimestamp' in event: - del event['receivedtimestamp'] - - # add demo to the tags so it's clear it's not real data. - if 'tags' not in event: - event['tags'] = list() - - event['tags'].append('demodata') - event['tags'].append('demoalert') - - # replace potential with a random ip address - if 'summary' in event and '' in event['summary']: - randomIP = genRandomIPv4() - event['summary'] = event['summary'].replace("", randomIP) - if 'details' not in event: - event['details'] = dict() - event['details']['sourceipaddress'] = randomIP - event['details']['sourceipv4address'] = randomIP - - if 'duplicate' in event: - # send this event multiple times to trigger an alert - for x in range(0, int(event['duplicate'])): - logcache.put(json.dumps(event)) - else: - logcache.put(json.dumps(event)) - lastalert=toUTC(datetime.now()).isoformat() - setConfig('lastalert',lastalert,options.configfile) - if not logcache.empty(): - time.sleep(.01) - try: - postingProcess=Process(target=postLogs,args=(logcache,),name="json2MozdefDemoData") - postingProcess.start() - except OSError as e: - if e.errno == 35: # resource temporarily unavailable. - print(e) - pass - else: - logger.error('%r' % e) - - except KeyboardInterrupt as e: - sys.exit(1) - - -def makeAttackers(): - ''' - send events that will be correlated into attackers using pre-defined IPs - ''' - try: - # time for us to run? - timetoRun=toUTC(options.lastattacker) + timedelta(minutes=options.attackersminutesinterval) - if timetoRun > toUTC(datetime.now()): - # print(timetoRun) - return - - # print(timetoRun, options.lastalert) - eventfiles = glob.glob(options.alertsglob) - # pick a random number of events to send - for i in range(0, options.alertscount): - # pick a random type of event to send - eventfile = random.choice(eventfiles) - events = json.load(open(eventfile)) - target = random.randint(0, len(events)) - # if there's only one event in the file..use it. - if len(events) == 1 and target == 1: - target = 0 - for event in events[target:target + 1]: - event['timestamp'] = pytz.timezone('UTC').localize(datetime.utcnow()).isoformat() - # remove stored times - if 'utctimestamp' in event: - del event['utctimestamp'] - if 'receivedtimestamp' in event: - del event['receivedtimestamp'] - - # add demo to the tags so it's clear it's not real data. - if 'tags' not in event: - event['tags'] = list() - - event['tags'].append('demodata') - event['tags'].append('demoalert') - - # replace potential with a random ip address - if 'summary' in event and '' in event['summary']: - randomIP = genAttackerIPv4() - event['summary'] = event['summary'].replace("", randomIP) - if 'details' not in event: - event['details'] = dict() - event['details']['sourceipaddress'] = randomIP - event['details']['sourceipv4address'] = randomIP - - if 'duplicate' in event: - # send this event multiple times to trigger an alert - for x in range(0, int(event['duplicate'])): - logcache.put(json.dumps(event)) - else: - logcache.put(json.dumps(event)) - lastattacker=toUTC(datetime.now()).isoformat() - setConfig('lastattacker',lastattacker,options.configfile) - if not logcache.empty(): - time.sleep(.01) - try: - postingProcess=Process(target=postLogs,args=(logcache,),name="json2MozdefDemoData") - postingProcess.start() - except OSError as e: - if e.errno == 35: # resource temporarily unavailable. - print(e) - pass - else: - logger.error('%r' % e) - - except KeyboardInterrupt as e: - sys.exit(1) - - -def initConfig(): - options.url = getConfig('url', 'http://localhost:8080/events/', options.configfile) - options.eventsglob = getConfig('eventsglob', './sampleevents/events*json', options.configfile) - options.alertsglob = getConfig('alertsglob', './sampleevents/alert*json', options.configfile) - options.attackersglob = getConfig('attackersglob', './sampleevents/attacker*json', options.configfile) - # how many alerts to create - options.alertscount = getConfig('alertscount', 2, options.configfile) - # how many minutes to wait between creating ^ alerts - options.alertsminutesinterval = getConfig('alertsminutesinterval', 5, options.configfile) - options.lastalert = getConfig('lastalert', datetime.now() - timedelta(hours=1), options.configfile) - - # how many attackers to create - options.attackerscount = getConfig('attackers', 1, options.configfile) - # how many minutes to wait between creating ^ attackers - options.attackersminutesinterval = getConfig('attackersminutesinterval', 5, options.configfile) - options.lastattacker = getConfig('lastattacker', datetime.now() - timedelta(hours=1), options.configfile) - - -if __name__ == '__main__': - parser = OptionParser() - parser.add_option("-c", - dest='configfile', - default=sys.argv[0].replace('.py', '.conf'), - help="configuration file to use") - (options, args) = parser.parse_args() - initConfig() - - sh=logging.StreamHandler(sys.stdout) - sh.setFormatter(formatter) - logger.addHandler(sh) - - makeEvents() - makeAlerts() - makeAttackers() - - while not logcache.empty(): - try: - postingProcess=Process(target=postLogs,args=(logcache,),name="json2MozdefDemoData") - postingProcess.start() - except OSError as e: - if e.errno == 35: # resource temporarily unavailable. - print(e) - pass - else: - logger.error('%r' % e) diff --git a/examples/demo/sampleevents.sh b/examples/demo/sampleevents.sh deleted file mode 100755 index caea4d7fe..000000000 --- a/examples/demo/sampleevents.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -while true -do - ./sampleData2MozDef.py - sleep 2 -done diff --git a/examples/es-docs/alerts-kibana.json b/examples/es-docs/alerts-kibana.json deleted file mode 100644 index e4c08f4ae..000000000 --- a/examples/es-docs/alerts-kibana.json +++ /dev/null @@ -1,287 +0,0 @@ -{ - "title": "MozDef Sample Alerts", - "services": { - "query": { - "idQueue": [ - 1, - 2, - 3, - 4 - ], - "list": { - "0": { - "query": "*", - "alias": "", - "color": "#7EB26D", - "id": 0, - "pin": false, - "type": "lucene", - "enable": true - } - }, - "ids": [ - 0 - ] - }, - "filter": { - "idQueue": [ - 0, - 1, - 2 - ], - "list": {}, - "ids": [] - } - }, - "rows": [ - { - "title": "Options", - "height": "50px", - "editable": true, - "collapse": false, - "collapsable": true, - "panels": [ - { - "error": false, - "span": 7, - "editable": true, - "group": [ - "default" - ], - "type": "text", - "status": "Stable", - "mode": "markdown", - "content": "If you have a field with a timestamp in it, you can set a time filter using the control in the navigation bar. You'll need to click the cog icon to configure the field that your timestamp is in.", - "style": {}, - "title": "Have a timestamp somewhere?" - }, - { - "error": false, - "span": 5, - "editable": true, - "group": [ - "default" - ], - "type": "text", - "status": "Stable", - "mode": "markdown", - "content": "See the *Filters* bar above? Click it to expand the filters panel. Right now there are none. click on one of the icons in the document types list to filter down to only that document type", - "style": {}, - "title": "About filters" - } - ], - "notice": false - }, - { - "title": "Graph", - "height": "250px", - "editable": true, - "collapse": false, - "collapsable": true, - "panels": [ - { - "error": false, - "span": 3, - "editable": true, - "group": [ - "default" - ], - "type": "terms", - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "field": "_type", - "exclude": [], - "missing": true, - "other": true, - "size": 100, - "order": "count", - "style": { - "font-size": "10pt" - }, - "donut": false, - "tilt": false, - "labels": true, - "arrangement": "horizontal", - "chart": "pie", - "counter_pos": "none", - "title": "Document types", - "spyable": true - }, - { - "error": false, - "span": 3, - "editable": true, - "group": [ - "default" - ], - "type": "terms", - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "field": "_type", - "exclude": [], - "missing": true, - "other": true, - "size": 10, - "order": "count", - "style": { - "font-size": "10pt" - }, - "donut": false, - "tilt": false, - "labels": true, - "arrangement": "horizontal", - "chart": "table", - "counter_pos": "above", - "spyable": true, - "title": "Document Types" - }, - { - "error": false, - "span": 6, - "editable": true, - "group": [ - "default" - ], - "type": "text", - "status": "Stable", - "mode": "markdown", - "content": "It's the best I can do without knowing much about your data! I've tried to pick some sane defaults for you. The two *terms* panels to the left of this *text* panel show a breakdown of your document type. \n\nKibana is currently configured to point at the special Elasticsearch *_all* index. You can change that by clicking on the cog icon in the navigation bar at the top. You can also add rows from that dialog. You can edit individual panels by click on the cog icon on the panel you want to edit\n\nThe *table* panel below has attempted to list your fields to the left, select a few to view them in the table. To add more panels, of different types, click the cog on the row label to the far left", - "style": {}, - "title": "The most generic dashboard ever" - } - ], - "notice": false - }, - { - "title": "Events", - "height": "650px", - "editable": true, - "collapse": false, - "collapsable": true, - "panels": [ - { - "error": false, - "span": 12, - "editable": true, - "group": [ - "default" - ], - "type": "table", - "size": 100, - "pages": 5, - "offset": 0, - "sort": [ - "_id", - "desc" - ], - "style": { - "font-size": "9pt" - }, - "overflow": "min-height", - "fields": [], - "highlight": [], - "sortable": true, - "header": true, - "paging": true, - "spyable": true, - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "field_list": true, - "status": "Stable", - "trimFactor": 300, - "normTimes": true, - "title": "Documents", - "all_fields": false - } - ], - "notice": false - } - ], - "editable": true, - "index": { - "interval": "none", - "pattern": "none", - "default": "alerts" - }, - "style": "dark", - "failover": false, - "panel_hints": true, - "loader": { - "save_gist": false, - "save_elasticsearch": true, - "save_local": true, - "save_default": true, - "save_temp": true, - "save_temp_ttl_enable": true, - "save_temp_ttl": "30d", - "load_gist": true, - "load_elasticsearch": true, - "load_elasticsearch_size": 20, - "load_local": true, - "hide": false - }, - "pulldowns": [ - { - "type": "query", - "collapse": false, - "notice": false, - "query": "*", - "pinned": true, - "history": [], - "remember": 10, - "enable": true - }, - { - "type": "filtering", - "collapse": true, - "notice": false, - "enable": true - } - ], - "nav": [ - { - "type": "timepicker", - "collapse": false, - "notice": false, - "status": "Stable", - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ], - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "timefield": "utctimestamp", - "enable": true - } - ], - "refresh": false -} \ No newline at end of file diff --git a/examples/es-docs/alerts_template.json b/examples/es-docs/alerts_template.json deleted file mode 100644 index 3cbb82154..000000000 --- a/examples/es-docs/alerts_template.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "template":"alerts*", - "mappings":{ - "alert":{ - "properties":{ - "timestamp":{ - "format":"dateOptionalTime", - "type":"date" - }, - "tags":{ - "type":"string" - }, - "summary":{ - "type":"string" - }, - "receivedtimestatmp":{ - "format":"dateOptionalTime", - "type":"date" - }, - "category":{ - "index":"not_analyzed", - "type":"string" - }, - "events":{ - "properties":{ - "id":{ - "type":"string" - }, - "index":{ - "type":"string" - }, - "type":{ - "type":"string" - } - } - }, - "eventsource":{ - "type":"string" - }, - "hostname":{ - "type":"string" - }, - "severity":{ - "type":"string" - }, - "utctimestamp":{ - "format":"dateOptionalTime", - "type":"date" - } - } - } - } -} \ No newline at end of file diff --git a/examples/es-docs/bro_intel.json b/examples/es-docs/bro_intel.json deleted file mode 100644 index ecbb10f1f..000000000 --- a/examples/es-docs/bro_intel.json +++ /dev/null @@ -1,170 +0,0 @@ -[ - { - "category": "brointel", - "processid": "0", - "receivedtimestamp": "2014-07-16T21:32:07.502716+00:00", - "severity": "7", - "utctimestamp": "2014-07-16T21:32:07+00:00", - "tags": "nsm,bro,intel", - "timestamp": "2014-07-16T21:32:07+00:00", - "hostname": "nsm5", - "summary": "Bro intel match: 0.0.139.213", - "eventsource": "nsm", - "details": { - "category": "brointel", - "destinationipaddress": "0.0.82.27", - "seenwhere": "Intel::ADDR", - "uid": "C4RdjhyE2jkvRH54d", - "ts": "1405546326.853474", - "seenindicator": "0.0.139.213", - "sources": "CIF - need-to-know", - "sourceipv4address": "0.0.82.208", - "seenindicatortype": "HTTP::IN_X_FORWARDED_FOR_HEADER", - "destinationport": 443, - "sourceport": 58969, - "sourceipaddress": "0.0.82.208", - "destinationipv4address": "0.0.82.27", - "severity": "NOTICE" - } - }, - { - "category": "brointel", - "processid": "0", - "receivedtimestamp": "2014-07-16T21:32:07.499594+00:00", - "severity": "7", - "utctimestamp": "2014-07-16T21:32:07+00:00", - "tags": "nsm,bro,intel", - "timestamp": "2014-07-16T21:32:07+00:00", - "hostname": "nsm5", - "summary": "Bro intel match: 0.0.139.213", - "eventsource": "nsm", - "details": { - "category": "brointel", - "destinationipaddress": "0.0.82.28", - "seenwhere": "Intel::ADDR", - "uid": "Ce58I13SIYMCYbcAw4", - "seenindicator": "0.0.139.213", - "ts": "1405546326.853474", - "sources": "CIF - need-to-know", - "sourceipv4address": "0.0.82.208", - "seenindicatortype": "HTTP::IN_X_FORWARDED_FOR_HEADER", - "destinationport": 443, - "sourceport": 13711, - "sourceipaddress": "0.0.82.208", - "destinationipv4address": "0.0.82.28", - "severity": "NOTICE" - } - }, - { - "category": "brointel", - "processid": "0", - "receivedtimestamp": "2014-07-16T21:32:07.499594+00:00", - "severity": "7", - "utctimestamp": "2014-07-16T21:32:07+00:00", - "tags": "nsm,bro,intel", - "timestamp": "2014-07-16T21:32:07+00:00", - "hostname": "nsm5", - "summary": "Bro intel match: 0.0.139.213", - "eventsource": "nsm", - "details": { - "category": "brointel", - "destinationipaddress": "0.0.82.28", - "seenwhere": "Intel::ADDR", - "uid": "Ce58I13SIYMCYbcAw4", - "seenindicator": "0.0.139.213", - "ts": "1405546326.853474", - "sources": "CIF - need-to-know", - "sourceipv4address": "0.0.82.208", - "seenindicatortype": "HTTP::IN_X_FORWARDED_FOR_HEADER", - "destinationport": 443, - "sourceport": 13711, - "sourceipaddress": "0.0.82.208", - "destinationipv4address": "0.0.82.28", - "severity": "NOTICE" - } - }, - { - "category": "brointel", - "processid": "0", - "receivedtimestamp": "2014-07-16T21:32:07.499594+00:00", - "severity": "7", - "utctimestamp": "2014-07-16T21:32:07+00:00", - "tags": "nsm,bro,intel", - "timestamp": "2014-07-16T21:32:07+00:00", - "hostname": "nsm5", - "summary": "Bro intel match: 0.0.139.213", - "eventsource": "nsm", - "details": { - "category": "brointel", - "destinationipaddress": "0.0.82.28", - "seenwhere": "Intel::ADDR", - "uid": "Ce58I13SIYMCYbcAw4", - "seenindicator": "0.0.139.213", - "ts": "1405546326.853474", - "sources": "CIF - need-to-know", - "sourceipv4address": "0.0.82.208", - "seenindicatortype": "HTTP::IN_X_FORWARDED_FOR_HEADER", - "destinationport": 443, - "sourceport": 13711, - "sourceipaddress": "0.0.82.208", - "destinationipv4address": "0.0.82.28", - "severity": "NOTICE" - } - }, - { - "category": "brointel", - "processid": "0", - "receivedtimestamp": "2014-07-16T21:32:07.499594+00:00", - "severity": "7", - "utctimestamp": "2014-07-16T21:32:07+00:00", - "tags": "nsm,bro,intel", - "timestamp": "2014-07-16T21:32:07+00:00", - "hostname": "nsm5", - "summary": "Bro intel match: 0.0.139.213", - "eventsource": "nsm", - "details": { - "category": "brointel", - "destinationipaddress": "0.0.82.28", - "seenwhere": "Intel::ADDR", - "uid": "Ce58I13SIYMCYbcAw4", - "seenindicator": "0.0.139.213", - "ts": "1405546326.853474", - "sources": "CIF - need-to-know", - "sourceipv4address": "0.0.82.208", - "seenindicatortype": "HTTP::IN_X_FORWARDED_FOR_HEADER", - "destinationport": 443, - "sourceport": 13711, - "sourceipaddress": "0.0.82.208", - "destinationipv4address": "0.0.82.28", - "severity": "NOTICE" - } - }, - { - "category": "brointel", - "processid": "0", - "receivedtimestamp": "2014-07-16T21:32:07.499594+00:00", - "severity": "7", - "utctimestamp": "2014-07-16T21:32:07+00:00", - "tags": "nsm,bro,intel", - "timestamp": "2014-07-16T21:32:07+00:00", - "hostname": "nsm5", - "summary": "Bro intel match: 0.0.139.213", - "eventsource": "nsm", - "details": { - "category": "brointel", - "destinationipaddress": "0.0.82.28", - "seenwhere": "Intel::ADDR", - "uid": "Ce58I13SIYMCYbcAw4", - "seenindicator": "0.0.139.213", - "ts": "1405546326.853474", - "sources": "CIF - need-to-know", - "sourceipv4address": "0.0.82.208", - "seenindicatortype": "HTTP::IN_X_FORWARDED_FOR_HEADER", - "destinationport": 443, - "sourceport": 13711, - "sourceipaddress": "0.0.82.208", - "destinationipv4address": "0.0.82.28", - "severity": "NOTICE" - } - } -] \ No newline at end of file diff --git a/examples/es-docs/bro_notice.json b/examples/es-docs/bro_notice.json deleted file mode 100644 index 603c3a122..000000000 --- a/examples/es-docs/bro_notice.json +++ /dev/null @@ -1,17 +0,0 @@ -[ - { - "category": "bronotice", - "processid": "0", - "receivedtimestamp": "2014-07-17T05:37:57.617362+00:00", - "severity": "NOTICE", - "utctimestamp": "2014-07-17T05:37:56+00:00", - "tags": "nsm,bro,notice", - "timestamp": "2014-07-17T05:37:56+00:00", - "hostname": "nsm7", - "summary": "MozillaHTTPErrors::Excessive_HTTP_Errors_Attacker Excessive HTTP errors for requests from 0.0.224.14 3 in 1.0 hr, eps: 0", - "eventsource": "nsm", - "details": { - "payload": "" - } - } -] \ No newline at end of file diff --git a/examples/es-docs/bruteforce_ssh.json b/examples/es-docs/bruteforce_ssh.json deleted file mode 100644 index 365e26ad3..000000000 --- a/examples/es-docs/bruteforce_ssh.json +++ /dev/null @@ -1,107 +0,0 @@ -[ -{ - "category": "syslog", - "processid": "0", - "receivedtimestamp": "2014-07-17T14:00:15.167032+00:00", - "severity": "NOTICE", - "utctimestamp": "2014-07-17T14:00:14+00:00", - "timestamp": "2014-07-17T14:00:14+00:00", - "hostname": "example.com", - "summary": "%-SSHD_LOGIN_FAILED: Login failed for user 'ch' from host '0.0.72.113'\n", - "eventsource": "systemslogs", - "details": { - "processid": "", - "sourceipv4address": "0.0.72.113", - "timestamp": "Jul 17 14:00:13", - "hostname": "example.com", - "program": "sshd", - "sourceipgeolocation": { - "city": null, - "region_code": null, - "area_code": 0, - "time_zone": "Europe/Rome", - "dma_code": 0, - "metro_code": null, - "country_code3": "ITA", - "latitude": 42.83330000000001, - "postal_code": null, - "longitude": 12.833300000000008, - "country_code": "IT", - "country_name": "Italy", - "continent": "EU" - }, - "sourceipaddress": "0.0.72.113", - "payload": "" - } - }, - { - "category": "syslog", - "processid": "0", - "receivedtimestamp": "2014-07-17T14:00:11.864097+00:00", - "severity": "NOTICE", - "utctimestamp": "2014-07-17T14:00:11+00:00", - "timestamp": "2014-07-17T14:00:11+00:00", - "hostname": "example.com", - "summary": "%-SSHD_LOGIN_FAILED: Login failed for user 'ch' from host '0.0.72.113'\n", - "eventsource": "systemslogs", - "details": { - "processid": "", - "sourceipv4address": "0.0.72.113", - "timestamp": "Jul 17 14:00:10", - "hostname": "example.com", - "program": "sshd", - "sourceipgeolocation": { - "city": null, - "region_code": null, - "area_code": 0, - "time_zone": "Europe/Rome", - "dma_code": 0, - "metro_code": null, - "country_code3": "ITA", - "latitude": 42.83330000000001, - "postal_code": null, - "longitude": 12.833300000000008, - "country_code": "IT", - "country_name": "Italy", - "continent": "EU" - }, - "sourceipaddress": "0.0.72.113", - "payload": "" - } - }, - { - "category": "syslog", - "processid": "0", - "receivedtimestamp": "2014-07-17T14:00:09.573817+00:00", - "severity": "NOTICE", - "utctimestamp": "2014-07-17T14:00:08+00:00", - "timestamp": "2014-07-17T14:00:08+00:00", - "hostname": "example.com", - "summary": "%-SSHD_LOGIN_FAILED: Login failed for user 'ch' from host '0.0.72.113'\n", - "eventsource": "systemslogs", - "details": { - "processid": "", - "sourceipv4address": "0.0.72.113", - "timestamp": "Jul 17 14:00:07", - "hostname": "example.com", - "program": "sshd", - "sourceipgeolocation": { - "city": null, - "region_code": null, - "area_code": 0, - "time_zone": "Europe/Rome", - "dma_code": 0, - "metro_code": null, - "country_code3": "ITA", - "latitude": 42.83330000000001, - "postal_code": null, - "longitude": 12.833300000000008, - "country_code": "IT", - "country_name": "Italy", - "continent": "EU" - }, - "sourceipaddress": "0.0.72.113", - "payload": "" - } - } -] diff --git a/examples/es-docs/complianceitems_template.json b/examples/es-docs/complianceitems_template.json deleted file mode 100644 index 612577dcb..000000000 --- a/examples/es-docs/complianceitems_template.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "template": "complianceitems*", - "mappings": { - "history": { - "_ttl": { - "enabled": true - }, - "properties": { - "target": { - "type": "multi_field", - "fields": { - "target": { - "type": "string" - }, - "raw": { - "type": "string", - "index": "not_analyzed" - } - } - } - } - }, - "last_known_state": { - "_ttl": { - "enabled": true - }, - "properties": { - "target": { - "type": "multi_field", - "fields": { - "target": { - "type": "string" - }, - "raw": { - "type": "string", - "index": "not_analyzed" - } - } - } - } - }, - "_default_": { - "_ttl": { - "enabled": true - } - } - } -} \ No newline at end of file diff --git a/examples/es-docs/events-auditd.json b/examples/es-docs/events-auditd.json deleted file mode 100644 index c343b2ae5..000000000 --- a/examples/es-docs/events-auditd.json +++ /dev/null @@ -1,153 +0,0 @@ -[ - { - "timestamp": "2014-04-17T05:31:33.631170+00:00", - "summary": "Unix Exec", - "receivedtimestatmp": "2014-04-17T05:33:33.637966+00:00", - "utctimestamp": "2014-04-17T05:33:33.631170+00:00", - "tags": [ - "example" - ], - "details": { - "auid": "0", - "parentprocess": "java", - "severity": "3", - "duid": "2317", - "truncated": "Yes", - "deviceversion": "2", - "auditkey": "exec ", - "devicevendor": "Unix", - "tty": "(none)", - "duser": "mapred", - "dproc": "/usr/java/jdk1.6.0_31/jre/bin/java", - "version": "0", - "command": "", - "signatureid": "EXECVE", - "suser": "root", - "msg": "", - "dhost": "node33.example.com", - "fname": "/usr/java/jdk1.6.0_31/jre/bin/java", - "deviceproduct": "auditd", - "name": "Unix Exec" - } - }, - { - "timestamp": "2014-04-17T05:32:33.509015+00:00", - "summary": "Unix Exec", - "receivedtimestatmp": "2014-04-17T05:32:33.510685+00:00", - "utctimestamp": "2014-04-17T05:32:33.509015+00:00", - "tags": [ - "example" - ], - "details": { - "auid": "4294967295", - "parentprocess": "sosreport", - "severity": "3", - "duid": "0", - "truncated": "No", - "deviceversion": "2", - "auditkey": "exec ", - "devicevendor": "Unix", - "tty": "(none)", - "duser": "root", - "dproc": "/bin/bash", - "version": "0", - "command": "/bin/sh -c /bin/ls -laR /dev", - "signatureid": "EXECVE", - "fname": "/bin/sh", - "msg": "", - "dhost": "web3.example.com", - "deviceproduct": "auditd", - "name": "Unix Exec" - } - }, - { - "timestamp": "2014-04-17T05:33:33.222518+00:00", - "summary": "Unix Exec", - "receivedtimestatmp": "2014-04-17T05:33:33.224129+00:00", - "utctimestamp": "2014-04-17T05:33:33.222518+00:00", - "tags": [ - "example" - ], - "details": { - "auid": "4294967295", - "parentprocess": "gmond", - "severity": "3", - "duid": "99", - "truncated": "No", - "deviceversion": "2", - "auditkey": "exec ", - "devicevendor": "Unix", - "tty": "(none)", - "duser": "nobody", - "dproc": "/bin/bash", - "version": "0", - "command": "/bin/sh -c netstat -t -a -n", - "signatureid": "EXECVE", - "fname": "/bin/sh", - "msg": "", - "dhost": "node27.example.com", - "deviceproduct": "auditd", - "name": "Unix Exec" - } - }, - { - "timestamp": "2014-04-17T05:34:33.069873+00:00", - "summary": "Unix Exec", - "receivedtimestatmp": "2014-04-17T05:34:33.079831+00:00", - "utctimestamp": "2014-04-17T05:34:33.069873+00:00", - "tags": [ - "example" - ], - "details": { - "auid": "4294967295", - "parentprocess": "sosreport", - "severity": "3", - "duid": "0", - "truncated": "No", - "deviceversion": "2", - "auditkey": "exec ", - "devicevendor": "Unix", - "tty": "(none)", - "duser": "root", - "dproc": "/bin/bash", - "version": "0", - "command": "/bin/sh -c /sbin/route -n", - "signatureid": "EXECVE", - "fname": "/bin/sh", - "msg": "", - "dhost": "web9.example.com", - "deviceproduct": "auditd", - "name": "Unix Exec" - } - }, - { - "timestamp": "2014-04-17T05:35:33.039246+00:00", - "summary": "Unix Exec", - "receivedtimestatmp": "2014-04-17T05:35:33.041927+00:00", - "utctimestamp": "2014-04-17T05:35:33.039246+00:00", - "tags": [ - "example" - ], - "details": { - "auid": "4294967295", - "parentprocess": "sosreport", - "severity": "3", - "duid": "0", - "truncated": "No", - "deviceversion": "2", - "auditkey": "exec ", - "devicevendor": "Unix", - "tty": "(none)", - "duser": "root", - "dproc": "/sbin/ifconfig", - "version": "0", - "command": "/sbin/ifconfig -a", - "signatureid": "EXECVE", - "fname": "/sbin/ifconfig", - "msg": "", - "dhost": "monitoring.example.com", - "deviceproduct": "auditd", - "name": "Unix Exec" - } - } -] \ No newline at end of file diff --git a/examples/es-docs/events-cloudtrail.json b/examples/es-docs/events-cloudtrail.json deleted file mode 100644 index dc0b8c1ba..000000000 --- a/examples/es-docs/events-cloudtrail.json +++ /dev/null @@ -1,75 +0,0 @@ -[ - { - "eventVersion": "1.01", - "eventID": "8d5840cf-6ne3-4947-be6b-cb5147856719", - "eventTime": "2014-04-17T06:32:05Z", - "utctimestamp": "2014-04-17T06:32:05+00:00", - "responseElements": null, - "awsRegion": "us-east-1", - "eventName": "DescribeInstances", - "userIdentity": { - "userName": "John", - "principalId": "XXXXXXXXXXXXXXXXXXXXX", - "accessKeyId": "XXXXXXXXXXXXXXXXXXXXX", - "type": "IAMUser", - "arn": "arn:aws:iam::646131927850:user/John", - "accountId": "646131927850" - }, - "eventSource": "ec2.amazonaws.com", - "requestID": "eaa5966a-5d22-43f2-b2bf-4930afe601a4", - "userAgent": "aws-sdk-dotnet/1.4.11.0 .NET Runtime/4.0 .NET Framework/4.0 OS/6.1.7601.65536", - "sourceIPAddress": "59.15.171.43", - "tags": [ - "example" - ] - }, - { - "eventVersion": "1.01", - "eventID": "329817d2-f95b-43e0-bca3-4f70fb79f4ce", - "eventTime": "2014-04-17T06:30:03Z", - "utctimestamp": "2014-04-17T06:30:03+00:00", - "awsRegion": "us-east-1", - "eventName": "DescribeTrails", - "userIdentity": { - "userName": "jeff", - "principalId": "XXXXXXXXXXXXXXXXXXXXX", - "accessKeyId": "XXXXXXXXXXXXXXXXXXXXX", - "type": "IAMUser", - "arn": "arn:aws:iam::656531927250:user/jeff", - "accountId": "656531926850" - }, - "eventSource": "cloudtrail.amazonaws.com", - "requestID": "e0007bf9-c6c2-11e3-94ae-9f9112d60855", - "userAgent": "Boto/2.23.0 Python/2.7.6 Linux/2.6.32-431.el6.x86_64", - "sourceIPAddress": "61.245.214.162", - "tags": [ - "example" - ] - }, - { - "eventVersion": "1.0", - "eventTime": "2014-04-17T06:14:35Z", - "utctimestamp": "2014-04-17T06:14:35+00:00", - "requestParameters": { - "securityGroupSet": {}, - "securityGroupIdSet": {}, - "filterSet": {} - }, - "awsRegion": "us-west-2", - "eventName": "DescribeSecurityGroups", - "userIdentity": { - "userName": "John", - "principalId": "XXXXXXXXXXXXXXXXXXXXX", - "accessKeyId": "XXXXXXXXXXXXXXXXXXXXX", - "type": "IAMUser", - "arn": "arn:aws:iam::646131927850:user/John", - "accountId": "646131927850" - }, - "eventSource": "ec2.amazonaws.com", - "userAgent": "aws-sdk-dotnet/1.4.11.0 .NET Runtime/4.0 .NET Framework/4.0 OS/6.1.7601.65536", - "sourceIPAddress": "59.15.171.43", - "tags": [ - "example" - ] - } -] \ No newline at end of file diff --git a/examples/es-docs/events-event.json b/examples/es-docs/events-event.json deleted file mode 100644 index ff29aefb1..000000000 --- a/examples/es-docs/events-event.json +++ /dev/null @@ -1,322 +0,0 @@ -[ - { - "category": "syslog", - "processid": "0", - "severity": "INFO", - "utctimestamp": "2014-04-17T06:06:54+00:00", - "timestamp": "2014-04-17T06:06:54+00:00", - "hostname": "syslog.example.com", - "receivedtimestatmp": "2014-04-17T06:06:54.618178+00:00", - "summary": "Did not receive identification string from 10.0.0.1\n", - "eventsource": "systemslogs", - "tags": [ - "example" - ], - "details": { - "processid": "4846", - "program": "sshd", - "hostname": "proxy", - "payload": "", - "timestamp": "Apr 17 06:06:53" - } - }, - { - "category": "syslog", - "processid": "0", - "severity": "INFO", - "utctimestamp": "2014-04-17T06:06:53+00:00", - "timestamp": "2014-04-17T06:06:53+00:00", - "hostname": "syslog.example.com", - "receivedtimestatmp": "2014-04-17T06:06:53.827106+00:00", - "summary": "Connection from 10.0.0.214 port 35783\n", - "eventsource": "systemslogs", - "tags": [ - "example" - ], - "details": { - "processid": "2520", - "program": "sshd", - "hostname": "git", - "payload": "", - "timestamp": "Apr 17 06:06:52" - } - }, - { - "category": "syslog", - "processid": "0", - "severity": "INFO", - "utctimestamp": "2014-04-17T06:06:52+00:00", - "timestamp": "2014-04-17T06:06:52+00:00", - "hostname": "syslog.example.com", - "receivedtimestatmp": "2014-04-17T06:06:52.825668+00:00", - "summary": "Did not receive identification string from 10.0.0.210\n", - "eventsource": "systemslogs", - "tags": [ - "example" - ], - "details": { - "processid": "1939", - "program": "sshd", - "hostname": "git", - "payload": "", - "timestamp": "Apr 17 06:06:51" - } - }, - { - "category": "syslog", - "processid": "0", - "severity": "INFO", - "utctimestamp": "2014-04-17T06:10:54+00:00", - "timestamp": "2014-04-17T06:10:54+00:00", - "hostname": "syslog.example.com", - "receivedtimestatmp": "2014-04-17T06:10:54.929854+00:00", - "summary": " nagios : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/usr/lib64/nagios/plugins/custom/check_auditd.sh\n", - "eventsource": "systemslogs", - "tags": [ - "example" - ], - "details": { - "processid": "", - "program": "sudo", - "hostname": "input", - "payload": "", - "timestamp": "Apr 17 06:10:54" - } - }, - { - "category": "syslog", - "processid": "0", - "severity": "INFO", - "utctimestamp": "2014-04-17T06:10:51+00:00", - "timestamp": "2014-04-17T06:10:51+00:00", - "hostname": "syslog.example.com", - "receivedtimestatmp": "2014-04-17T06:10:51.866868+00:00", - "summary": " nagios : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/usr/lib64/nagios/plugins/custom/check_puppet -t 7200\n", - "eventsource": "systemslogs", - "tags": [ - "example" - ], - "details": { - "processid": "", - "program": "sudo", - "hostname": "redis", - "payload": "", - "timestamp": "Apr 17 06:10:51" - } - }, - { - "category": "syslog", - "processid": "0", - "severity": "INFO", - "utctimestamp": "2014-04-17T06:10:28+00:00", - "timestamp": "2014-04-17T06:10:28+00:00", - "hostname": "syslog.example.com", - "receivedtimestatmp": "2014-04-17T06:10:28.687338+00:00", - "summary": "named-update : TTY=unknown ; PWD=/var/named ; USER=named-update ; COMMAND=/usr/bin/svn status /var/named/chroot/var/named\n", - "eventsource": "systemslogs", - "tags": [ - "example" - ], - "details": { - "processid": "", - "program": "sudo", - "hostname": "admin", - "payload": "", - "timestamp": "Apr 17 06:10:27" - } - }, - { - "category": "network", - "processid": "0", - "severity": "INFO", - "utctimestamp": "2014-04-17T06:15:07+00:00", - "timestamp": "2014-04-17T06:15:07+00:00", - "hostname": "syslog.example.com", - "receivedtimestatmp": "2014-04-17T06:15:07.862013+00:00", - "summary": "%-UI_JUNOSCRIPT_CMD: User 'root' used JUNOScript client to run command 'get-flow-statistics-all'\n", - "eventsource": "networklogs", - "tags": [ - "example" - ], - "details": { - "processid": "35029", - "program": "mgd", - "hostname": "fw.example.com", - "payload": "", - "timestamp": "Apr 17 06:15:06" - } - }, - { - "category": "network", - "processid": "0", - "severity": "INFO", - "utctimestamp": "2014-04-17T06:19:41+00:00", - "timestamp": "2014-04-17T06:19:41+00:00", - "hostname": "syslog.example.com", - "receivedtimestatmp": "2014-04-17T06:19:41.957329+00:00", - "summary": "%-UI_LOGIN_EVENT: User 'root' login, class 'super-user' [744], ssh-connection '', client-mode 'junoscript'\n", - "eventsource": "networklogs", - "tags": [ - "example" - ], - "details": { - "processid": "744", - "program": "mgd", - "hostname": "switch1.example.com", - "payload": "", - "timestamp": "Apr 17 06:19:40" - } - }, - { - "utctimestamp": "2014-04-17T07:05:02+00:00", - "tags": [ - "example" - ], - "timestamp": "2014-04-17T00:05:02", - "receivedtimestatmp": "2014-04-17T00:05:03.941186+00:00", - "summary": "LDAP_SUCCESS john@example.com,o=com,dc=example srcIP=10.0.0.209", - "details": { - "dn": "john@example.com,o=com,dc=example", - "source": "Apr 17 00:05:02 ldap slapd[15932]: conn=25123333 fd=93 ACCEPT from IP=10.0.0.209:8325 (IP=0.0.0.0:389)\nApr 17 00:05:02 ldap slapd[15932]: conn=25123333 op=0 BIND dn=\"uid=bind-openvpn,ou=logins,dc=example\" method=128\nApr 17 00:05:02 ldap slapd[15932]: conn=25123333 op=0 BIND dn=\"uid=vpn,ou=logins,dc=example\" mech=SIMPLE ssf=0\nApr 17 00:05:02 ldap slapd[15932]: conn=25123333 op=0 RESULT tag=97 err=0 text=\nApr 17 00:05:02 ldap slapd[15932]: conn=25123333 op=1 BIND anonymous mech=implicit ssf=0\nApr 17 00:05:02 ldap slapd[15932]: conn=25123333 op=1 BIND dn=\"mail=john@example.com,o=com,dc=example\" method=128\nApr 17 00:05:02 ldap slapd[15932]: conn=25123333 op=1 BIND dn=\"mail=john@example.com,o=com,dc=example\" mech=SIMPLE ssf=0\nApr 17 00:05:02 ldap slapd[15932]: conn=25123333 op=1 RESULT tag=97 err=0 text=\n", - "srcip": "10.0.0.209", - "result": "LDAP_SUCCESS", - "success": true - } - }, - { - "category": "bronotice", - "processid": "0", - "severity": "NOTICE", - "utctimestamp": "2014-04-17T07:17:09+00:00", - "timestamp": "2014-04-17T07:17:09+00:00", - "hostname": "nsm5", - "receivedtimestatmp": "2014-04-17T07:17:10.634904+00:00", - "summary": "SSL::Invalid_Server_Cert SSL certificate validation failed with (unable to get local issuer certificate) emailAddress=john@example.com,CN=mail.example.com,OU=Secure Mail Server,O=Example Corporation,L=Mountain View,ST=California,C=US", - "eventsource": "nsm", - "tags": [ - "example" - ], - "details": { - "destinationipaddress": "10.0.0.38", - "uid": "CXOBsx4vMrhPXR4qM4", - "proto": "tcp", - "ts": "1397805429.043383", - "note": "SSL::Invalid_Server_Cert", - "sourceport": "46823", - "destinationport": "7071", - "msg": "SSL certificate validation failed with (unable to get local issuer certificate)", - "sourceipaddress": "10.0.0.154", - "payload": "", - "sub": "emailAddress=john@example.com,CN=mail.example.com,OU=Secure Mail Server,O=Example Corporation,L=Mountain View,ST=California,C=US" - } - }, - { - "category": "bronotice", - "processid": "0", - "severity": "NOTICE", - "utctimestamp": "2014-04-17T07:17:07+00:00", - "timestamp": "2014-04-17T07:17:07+00:00", - "hostname": "nsm5", - "receivedtimestatmp": "2014-04-17T07:17:08.674456+00:00", - "summary": "SSL::Invalid_Server_Cert SSL certificate validation failed with (unable to get local issuer certificate) CN=ssl-selfsigned-unknownissuer.example.com", - "eventsource": "nsm", - "tags": [ - "example" - ], - "details": { - "destinationipaddress": "8.8.8.8", - "destinationiplocation": "United States/San Francisco, CA", - "uid": "C5L6pJ2db92s2ajfnb", - "proto": "tcp", - "ts": "1397805427.078946", - "note": "SSL::Invalid_Server_Cert", - "sourceport": "34262", - "destinationport": "443", - "msg": "SSL certificate validation failed with (unable to get local issuer certificate)", - "sourceipaddress": "10.0.0.42", - "payload": "", - "sub": "CN=ssl-selfsigned-unknownissuer.example.com" - } - }, - { - "category": "bronotice", - "processid": "0", - "severity": "NOTICE", - "utctimestamp": "2014-04-17T07:16:37+00:00", - "timestamp": "2014-04-17T07:16:37+00:00", - "hostname": "nsm5", - "receivedtimestatmp": "2014-04-17T07:16:38.513274+00:00", - "summary": "SSH::Password_Guessing 10.0.0.78 appears to be guessing SSH passwords (seen in 30 connections). Sampled servers: 10.30.252.148, 10.30.252.150, 10.30.252.148, 10.30.252.150, 10.30.252.150", - "eventsource": "nsm", - "tags": [ - "example" - ], - "details": { - "destinationipaddress": "0.0.0.0", - "uid": "-", - "proto": "-", - "ts": "1397805396.838051", - "note": "SSH::Password_Guessing", - "sourceport": "-", - "destinationport": "-", - "msg": "10.0.0.78 appears to be guessing SSH passwords (seen in 30 connections).", - "sourceipaddress": "0.0.0.0", - "payload": "", - "sub": "Sampled servers: 10.30.252.148, 10.30.252.150, 10.30.252.148, 10.30.252.150, 10.30.252.150" - } - }, - { - "category": "bronotice", - "processid": "0", - "severity": "NOTICE", - "utctimestamp": "2014-04-17T07:16:36+00:00", - "timestamp": "2014-04-17T07:16:36+00:00", - "hostname": "nsm5", - "receivedtimestatmp": "2014-04-17T07:16:37.437511+00:00", - "summary": "SSH::Password_Guessing 10.0.0.7 appears to be guessing SSH passwords (seen in 35 connections). Sampled servers: 10.245.217.46, 10.245.217.46, 10.245.217.46, 10.245.217.46, 10.245.217.46", - "eventsource": "nsm", - "tags": [ - "example" - ], - "details": { - "destinationipaddress": "0.0.0.0", - "uid": "-", - "proto": "-", - "ts": "1397805396.486722", - "note": "SSH::Password_Guessing", - "sourceport": "-", - "destinationport": "-", - "msg": "10.0.0.7 appears to be guessing SSH passwords (seen in 35 connections).", - "sourceipaddress": "0.0.0.0", - "payload": "", - "sub": "Sampled servers: 10.245.217.46, 10.245.217.46, 10.245.217.46, 10.245.217.46, 10.245.217.46" - } - }, - { - "category": "bronotice", - "processid": "0", - "severity": "NOTICE", - "utctimestamp": "2014-04-17T07:06:34+00:00", - "timestamp": "2014-04-17T07:06:34+00:00", - "hostname": "nsm5", - "receivedtimestatmp": "2014-04-17T07:06:35.451657+00:00", - "summary": "SSL::Certificate_Expired Certificate CN=example.com expired at 2013-02-01-21:38:50.000000000 -", - "eventsource": "nsm", - "tags": [ - "example" - ], - "details": { - "destinationipaddress": "10.0.0.170", - "uid": "CAz8qn41YD9T8eNuh1", - "proto": "tcp", - "ts": "1397804793.952344", - "note": "SSL::Certificate_Expired", - "sourceport": "39764", - "destinationport": "311", - "msg": "Certificate CN=example.com expired at 2013-02-01-21:38:50.000000000", - "sourceipaddress": "10.0.0.128", - "payload": "", - "sub": "-" - } - } -] \ No newline at end of file diff --git a/examples/es-docs/events-kibana.json b/examples/es-docs/events-kibana.json deleted file mode 100644 index 3a86ada51..000000000 --- a/examples/es-docs/events-kibana.json +++ /dev/null @@ -1,287 +0,0 @@ -{ - "title": "MozDef Sample Events", - "services": { - "query": { - "idQueue": [ - 1, - 2, - 3, - 4 - ], - "list": { - "0": { - "query": "*", - "alias": "", - "color": "#7EB26D", - "id": 0, - "pin": false, - "type": "lucene", - "enable": true - } - }, - "ids": [ - 0 - ] - }, - "filter": { - "idQueue": [ - 0, - 1, - 2 - ], - "list": {}, - "ids": [] - } - }, - "rows": [ - { - "title": "Options", - "height": "50px", - "editable": true, - "collapse": false, - "collapsable": true, - "panels": [ - { - "error": false, - "span": 7, - "editable": true, - "group": [ - "default" - ], - "type": "text", - "status": "Stable", - "mode": "markdown", - "content": "If you have a field with a timestamp in it, you can set a time filter using the control in the navigation bar. You'll need to click the cog icon to configure the field that your timestamp is in.", - "style": {}, - "title": "Have a timestamp somewhere?" - }, - { - "error": false, - "span": 5, - "editable": true, - "group": [ - "default" - ], - "type": "text", - "status": "Stable", - "mode": "markdown", - "content": "See the *Filters* bar above? Click it to expand the filters panel. Right now there are none. click on one of the icons in the document types list to filter down to only that document type", - "style": {}, - "title": "About filters" - } - ], - "notice": false - }, - { - "title": "Graph", - "height": "250px", - "editable": true, - "collapse": false, - "collapsable": true, - "panels": [ - { - "error": false, - "span": 3, - "editable": true, - "group": [ - "default" - ], - "type": "terms", - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "field": "_type", - "exclude": [], - "missing": true, - "other": true, - "size": 100, - "order": "count", - "style": { - "font-size": "10pt" - }, - "donut": false, - "tilt": false, - "labels": true, - "arrangement": "horizontal", - "chart": "pie", - "counter_pos": "none", - "title": "Document types", - "spyable": true - }, - { - "error": false, - "span": 3, - "editable": true, - "group": [ - "default" - ], - "type": "terms", - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "field": "_type", - "exclude": [], - "missing": true, - "other": true, - "size": 10, - "order": "count", - "style": { - "font-size": "10pt" - }, - "donut": false, - "tilt": false, - "labels": true, - "arrangement": "horizontal", - "chart": "table", - "counter_pos": "above", - "spyable": true, - "title": "Document Types" - }, - { - "error": false, - "span": 6, - "editable": true, - "group": [ - "default" - ], - "type": "text", - "status": "Stable", - "mode": "markdown", - "content": "It's the best I can do without knowing much about your data! I've tried to pick some sane defaults for you. The two *terms* panels to the left of this *text* panel show a breakdown of your document type. \n\nKibana is currently configured to point at the special Elasticsearch *_all* index. You can change that by clicking on the cog icon in the navigation bar at the top. You can also add rows from that dialog. You can edit individual panels by click on the cog icon on the panel you want to edit\n\nThe *table* panel below has attempted to list your fields to the left, select a few to view them in the table. To add more panels, of different types, click the cog on the row label to the far left", - "style": {}, - "title": "The most generic dashboard ever" - } - ], - "notice": false - }, - { - "title": "Events", - "height": "650px", - "editable": true, - "collapse": false, - "collapsable": true, - "panels": [ - { - "error": false, - "span": 12, - "editable": true, - "group": [ - "default" - ], - "type": "table", - "size": 100, - "pages": 5, - "offset": 0, - "sort": [ - "_id", - "desc" - ], - "style": { - "font-size": "9pt" - }, - "overflow": "min-height", - "fields": [], - "highlight": [], - "sortable": true, - "header": true, - "paging": true, - "spyable": true, - "queries": { - "mode": "all", - "ids": [ - 0 - ] - }, - "field_list": true, - "status": "Stable", - "trimFactor": 300, - "normTimes": true, - "title": "Documents", - "all_fields": false - } - ], - "notice": false - } - ], - "editable": true, - "index": { - "interval": "none", - "pattern": "none", - "default": "events" - }, - "style": "dark", - "failover": false, - "panel_hints": true, - "loader": { - "save_gist": false, - "save_elasticsearch": true, - "save_local": true, - "save_default": true, - "save_temp": true, - "save_temp_ttl_enable": true, - "save_temp_ttl": "30d", - "load_gist": true, - "load_elasticsearch": true, - "load_elasticsearch_size": 20, - "load_local": true, - "hide": false - }, - "pulldowns": [ - { - "type": "query", - "collapse": false, - "notice": false, - "query": "*", - "pinned": true, - "history": [], - "remember": 10, - "enable": true - }, - { - "type": "filtering", - "collapse": true, - "notice": false, - "enable": true - } - ], - "nav": [ - { - "type": "timepicker", - "collapse": false, - "notice": false, - "status": "Stable", - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ], - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "timefield": "utctimestamp", - "enable": true - } - ], - "refresh": false -} \ No newline at end of file diff --git a/examples/es-docs/events_template.json b/examples/es-docs/events_template.json deleted file mode 100644 index 03479679b..000000000 --- a/examples/es-docs/events_template.json +++ /dev/null @@ -1,219 +0,0 @@ -{ - "template":"events*", - "mappings":{ - "event":{ - "_ttl" : { "enabled" : true }, - "properties":{ - "category":{ - "index":"not_analyzed", - "type":"string" - }, - "details":{ - "properties":{ - "destinationipaddress":{ - "type":"ip" - }, - "destinationport":{ - "type":"string" - }, - "dn":{ - "type":"string" - }, - "hostname":{ - "type" : "multi_field", - "fields" : { - "hostname": {"type": "string"}, - "raw" : {"type" : "string", "index" : "not_analyzed"} - } - }, - "email" : { - "type" : "string", - "index" : "not_analyzed" - }, - "msg":{ - "type":"string" - }, - "note":{ - "type":"string" - }, - "processid":{ - "type":"string" - }, - "program":{ - "type":"string", - "index" : "not_analyzed" - }, - "protocol":{ - "type":"string" - }, - "result":{ - "type":"string" - }, - "source":{ - "type":"string" - }, - "sourceipaddress":{ - "type":"ip" - }, - "sourceipgeolocation":{ - "properties":{ - "country_name": { - "type": "string", - "index" : "not_analyzed" - } - } - }, - "sourceport":{ - "type":"string" - }, - "srcip":{ - "type":"ip" - }, - "sub":{ - "type":"string" - }, - "success":{ - "type":"boolean" - }, - "timestamp":{ - "type":"string" - }, - "ts":{ - "type":"string" - }, - "uid":{ - "type":"string" - } - } - }, - "eventsource":{ - "type":"string" - }, - "hostname":{ - "type":"string" - }, - "processid":{ - "type":"string" - }, - "receivedtimestamp":{ - "type":"date", - "format":"dateOptionalTime" - }, - "severity":{ - "type":"string" - }, - "summary":{ - "type":"string" - }, - "tags":{ - "index":"not_analyzed", - "type":"string" - }, - "timestamp":{ - "type":"date", - "format":"dateOptionalTime" - }, - "utctimestamp":{ - "type":"date", - "format":"dateOptionalTime" - } - } - }, - "auditd":{ - "_ttl" : { "enabled" : true }, - "properties":{ - "category":{ - "index":"not_analyzed", - "type":"string" - }, - "details":{ - "properties":{ - "dhost":{ - "type" : "multi_field", - "fields" : { - "dhost": {"type": "string"}, - "raw" : {"type" : "string", "index" : "not_analyzed"} - } - }, - "auid":{ - "type":"long" - }, - "deviceversion":{ - "type": "integer" - }, - "duid":{ - "type": "long" - }, - "egid":{ - "type": "long" - }, - "euid":{ - "type": "long" - }, - "fsgid":{ - "type": "long" - }, - "fsuid":{ - "type": "long" - }, - "gid":{ - "type": "long" - }, - "ses":{ - "type": "long" - }, - "severity":{ - "type": "integer" - }, - "sgid":{ - "type": "long" - }, - "suid":{ - "type": "long" - }, - "version":{ - "type": "integer" - }, - "ogid": { - "type": "long" - }, - "ouid": { - "type": "long" - }, - "uid": { - "type": "long" - }, - "pid": { - "type": "long" - } - } - }, - "receivedtimestamp":{ - "type":"date", - "format":"dateOptionalTime" - }, - "severity":{ - "type":"string" - }, - "summary":{ - "type":"string" - }, - "tags":{ - "index":"not_analyzed", - "type":"string" - }, - "timestamp":{ - "type":"date", - "format":"dateOptionalTime" - }, - "utctimestamp":{ - "type":"date", - "format":"dateOptionalTime" - } - } - }, - "_default_": { - "_ttl" : { "enabled" : true } - } - } -} diff --git a/examples/es-docs/fail2ban.json b/examples/es-docs/fail2ban.json deleted file mode 100644 index 162c2279b..000000000 --- a/examples/es-docs/fail2ban.json +++ /dev/null @@ -1,37 +0,0 @@ -[ - { - "category": "syslog", - "processid": "0", - "receivedtimestamp": "2014-07-17T15:57:19.358119+00:00", - "severity": "WARNING", - "utctimestamp": "2014-07-17T15:57:18+00:00", - "timestamp": "2014-07-17T15:57:18+00:00", - "hostname": "example.com", - "summary": "The IP 0.0.141.210 has been banned for 21600 seconds after 4 failed attempts against SIP\n", - "eventsource": "systemslogs", - "details": { - "processid": "", - "sourceipv4address": "0.0.141.210", - "timestamp": "Jul 17 15:57:16", - "hostname": "pbx1", - "program": "fail2ban", - "sourceipgeolocation": { - "city": "Buffalo", - "region_code": "NY", - "area_code": 716, - "time_zone": "America/New_York", - "dma_code": 514, - "metro_code": "Buffalo, NY", - "country_code3": "USA", - "latitude": 42.9864, - "postal_code": "14221", - "longitude": -78.7279, - "country_code": "US", - "country_name": "United States", - "continent": "NA" - }, - "sourceipaddress": "0.0.141.210", - "payload": "" - } - } -] diff --git a/examples/es-docs/simpleDefaultTemplate.py b/examples/es-docs/simpleDefaultTemplate.py deleted file mode 100644 index 7b35eca98..000000000 --- a/examples/es-docs/simpleDefaultTemplate.py +++ /dev/null @@ -1,75 +0,0 @@ -import requests -import json - -# create a default template -defaultTemplate = r''' -{ - "template" : "*", - "mappings" : { - "_default_" : { - "dynamic_templates" : [ - { - "string_fields" : { - "mapping" : { - "index" : "not_analyzed", - "type" : "string", - "doc_values": true - }, - "match_mapping_type" : "string", - "match" : "*" - } - } - ], - - "properties" : { - "utctimestamp":{ - "type":"date", - "format":"dateOptionalTime" - }, - "receivedtimestamp":{ - "type":"date", - "format":"dateOptionalTime" - }, - "summary":{ - "type":"string" - }, - "details":{ - "properties":{ - "destinationipaddress":{ - "type":"ip" - }, - "sourceipaddress":{ - "type":"ip" - }, - "srcip":{ - "type":"ip" - }, - "success":{ - "type":"boolean" - }, - "sourceport":{ - "type":"long", - "index": "not_analyzed" - }, - "destinationport":{ - "type":"long", - "index": "not_analyzed" - } - } - } - }, - - "_all" : { - "enabled" : true - } - } - } -} -''' - -# valid json? -templateJson = json.loads(defaultTemplate) - -# post it -r = requests.put(url="http://servername:9200/_template/defaulttemplate", data=defaultTemplate) -print(r) diff --git a/examples/es-docs/vulnsnitch_template.json b/examples/es-docs/vulnsnitch_template.json deleted file mode 100644 index f4fd30db8..000000000 --- a/examples/es-docs/vulnsnitch_template.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "template": "vulnsnitch*", - "mappings": { - "history": { - "_ttl": { - "enabled": true - }, - "properties": { - "advisory_id": { - "index": "not_analyzed", - "type": "string" - }, - "cve": { - "index": "not_analyzed", - "type": "string" - }, - "host": { - "properties": { - "hostname": { - "type": "multi_field", - "fields": { - "hostname": { - "type": "string" - }, - "raw": { - "type": "string", - "index": "not_analyzed" - } - } - } - } - }, - "packages": { - "properties": { - "arch": { - "type": "string", - "index": "not_analyzed" - }, - "name": { - "type": "string", - "index": "not_analyzed" - } - } - }, - "totals": { - "properties": { - "advisories": { - "type": "integer" - }, - "cve": { - "type": "integer" - }, - "packages": { - "type": "integer" - } - } - } - } - }, - "last_known_state": { - "_ttl": { - "enabled": true - }, - "properties": { - "cve": { - "index": "not_analyzed", - "type": "string" - }, - "host": { - "properties": { - "hostname": { - "type": "multi_field", - "fields": { - "hostname": { - "type": "string" - }, - "raw": { - "type": "string", - "index": "not_analyzed" - } - } - } - } - }, - "packages": { - "properties": { - "arch": { - "type": "string", - "index": "not_analyzed" - }, - "name": { - "type": "string", - "index": "not_analyzed" - } - } - }, - "totals": { - "properties": { - "advisories": { - "type": "integer" - }, - "cve": { - "type": "integer" - }, - "packages": { - "type": "integer" - } - } - } - } - }, - "_default_": { - "_ttl": { - "enabled": true - } - } - } -} \ No newline at end of file diff --git a/scripts/demo/populate_sample_events.py b/scripts/demo/populate_sample_events.py new file mode 100644 index 000000000..5e5acb6f0 --- /dev/null +++ b/scripts/demo/populate_sample_events.py @@ -0,0 +1,69 @@ +import glob +import os +import optparse +import random +import hjson +import time +from datetime import datetime + +from mozdef_util.utilities.toUTC import toUTC +from mozdef_util.elasticsearch_client import ElasticsearchClient + + +def handle_event(event): + timestamp = toUTC(datetime.now()).isoformat() + event['timestamp'] = timestamp + event['receivedtimestamp'] = timestamp + event['utctimestamp'] = timestamp + + # add demo to the tags so it's clear it's not real data. + if 'tags' not in event: + event['tags'] = list() + event['tags'] += 'demodata' + return event + + +def handle_events(sample_events, num_picked, es_client): + selected_events = [] + if num_picked == 0: + selected_events = sample_events + else: + # pick a random type of event to send + for i in range(0, num_picked): + selected_events.append(random.choice(sample_events)) + for event in selected_events: + event = handle_event(event) + es_client.save_event(event) + + +def run(num_rounds, num_events, sleep_time, es_client): + sample_events_dir = os.path.join(os.path.dirname(__file__), "sample_events") + sample_event_files = glob.glob(sample_events_dir + '/*') + sample_events = [] + for sample_file in sample_event_files: + sample_events += hjson.load(open(sample_file)) + # # pick a random number of events to send + if num_rounds == 0: + print("Running indefinitely") + while True: + handle_events(sample_events, num_events, es_client) + time.sleep(sleep_time) + else: + print("Running for {0} rounds".format(num_rounds)) + handle_events(sample_events, num_events, es_client) + + +if __name__ == '__main__': + parser = optparse.OptionParser() + parser.add_option('--elasticsearch_host', help='Elasticsearch host (default: http://localhost:9200)', default='http://localhost:9200') + parser.add_option('--num_events', help='Number of random events to insert (default: 0 (run all))', default=0) + parser.add_option('--num_rounds', help='Number of rounds to insert events (default: 0 (run continuously))', default=0) + parser.add_option('--sleep_time', help='Number of seconds to sleep between rounds (default: 2)', default=2) + options, arguments = parser.parse_args() + es_client = ElasticsearchClient(options.elasticsearch_host) + run( + num_rounds=options.num_rounds, + num_events=options.num_events, + sleep_time=options.sleep_time, + es_client=es_client + ) diff --git a/scripts/demo/populate_sample_events.sh b/scripts/demo/populate_sample_events.sh new file mode 100644 index 000000000..447f92cda --- /dev/null +++ b/scripts/demo/populate_sample_events.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# Copyright (c) 2016 Mozilla Corporation + + +source /opt/mozdef/envs/python/bin/activate +/opt/mozdef/envs/mozdef/scripts/demo/populate_sample_events.py diff --git a/examples/demo/sampleevents/alertcreating-bro-intel.json b/scripts/demo/sample_events/alertcreating-bro-intel.json similarity index 100% rename from examples/demo/sampleevents/alertcreating-bro-intel.json rename to scripts/demo/sample_events/alertcreating-bro-intel.json diff --git a/examples/demo/sampleevents/alertcreating-bro-notice.json b/scripts/demo/sample_events/alertcreating-bro-notice.json similarity index 100% rename from examples/demo/sampleevents/alertcreating-bro-notice.json rename to scripts/demo/sample_events/alertcreating-bro-notice.json diff --git a/examples/demo/sampleevents/alertcreating-bruteforce-ssh.json b/scripts/demo/sample_events/alertcreating-bruteforce-ssh.json similarity index 100% rename from examples/demo/sampleevents/alertcreating-bruteforce-ssh.json rename to scripts/demo/sample_events/alertcreating-bruteforce-ssh.json diff --git a/examples/demo/sampleevents/alertcreating-fail2ban.json b/scripts/demo/sample_events/alertcreating-fail2ban.json similarity index 100% rename from examples/demo/sampleevents/alertcreating-fail2ban.json rename to scripts/demo/sample_events/alertcreating-fail2ban.json diff --git a/examples/demo/sampleevents/events-auditd.json b/scripts/demo/sample_events/events-auditd.json similarity index 100% rename from examples/demo/sampleevents/events-auditd.json rename to scripts/demo/sample_events/events-auditd.json diff --git a/examples/demo/sampleevents/events-benign.json b/scripts/demo/sample_events/events-benign.json similarity index 100% rename from examples/demo/sampleevents/events-benign.json rename to scripts/demo/sample_events/events-benign.json diff --git a/examples/demo/sampleevents/events-cloudtrail.json b/scripts/demo/sample_events/events-cloudtrail.json similarity index 100% rename from examples/demo/sampleevents/events-cloudtrail.json rename to scripts/demo/sample_events/events-cloudtrail.json diff --git a/examples/demo/sampleevents/events-logins-failure.json b/scripts/demo/sample_events/events-logins-failure.json similarity index 100% rename from examples/demo/sampleevents/events-logins-failure.json rename to scripts/demo/sample_events/events-logins-failure.json diff --git a/examples/demo/sampleevents/events-logins-success.json b/scripts/demo/sample_events/events-logins-success.json similarity index 100% rename from examples/demo/sampleevents/events-logins-success.json rename to scripts/demo/sample_events/events-logins-success.json diff --git a/examples/demo/sampleevents/events-network.json b/scripts/demo/sample_events/events-network.json similarity index 100% rename from examples/demo/sampleevents/events-network.json rename to scripts/demo/sample_events/events-network.json