From 0c6c4b65a6c0f2051d074e87bbb2da2424fa6c35 Mon Sep 17 00:00:00 2001 From: Daniel White Date: Fri, 17 Jan 2020 12:04:44 +0100 Subject: [PATCH] Initial Sigma support (#1028) * Initial work on Sigma support * Initial work on Sigma support * WIP * Initial work on Sigma support * Initial work on Sigma support * Copy sigma config into docker container * squash! Copy sigma config into docker container * squash! Copy sigma config into docker container * Update requirements.txt Co-authored-by: Johan Berggren --- data/linux/recon_commands.yaml | 24 ++++ data/linux/reverse_shell.yaml | 13 ++ data/sigma_config.yaml | 30 +++++ docker/Dockerfile | 4 +- requirements.txt | 3 +- setup.py | 2 +- timesketch/lib/analyzers/__init__.py | 1 + timesketch/lib/analyzers/interface.py | 29 ++++- timesketch/lib/analyzers/sigma_tagger.py | 116 ++++++++++++++++++ timesketch/lib/analyzers/sigma_tagger_test.py | 28 +++++ 10 files changed, 241 insertions(+), 9 deletions(-) create mode 100644 data/linux/recon_commands.yaml create mode 100644 data/linux/reverse_shell.yaml create mode 100644 data/sigma_config.yaml create mode 100644 timesketch/lib/analyzers/sigma_tagger.py create mode 100644 timesketch/lib/analyzers/sigma_tagger_test.py diff --git a/data/linux/recon_commands.yaml b/data/linux/recon_commands.yaml new file mode 100644 index 0000000000..bfa321e4ff --- /dev/null +++ b/data/linux/recon_commands.yaml @@ -0,0 +1,24 @@ +title: Linux reconnaissance commands +description: Commands that are run by attackers after compromising a system +logsource: + service: shell +references: + - https://github.com/mubix/post-exploitation/wiki/Linux-Post-Exploitation-Command-List +detection: + keywords: + - 'uname -a' + - 'cat /proc/version' + - 'grep pass' + - 'getent group' + - 'getent passwd' + - 'cat /home/*/.ssh/authorized_keys' + - 'cat /etc/sudoers' + - 'cat /etc/passwd' + - 'cat /etc/resolv.conf' + - 'ps aux' + - 'who -a' + - 'hostname -f' + - 'netstat -nltupw' + - 'cat /proc/net/*' +timeframe: 30m +conditions: count > 3 \ No newline at end of file diff --git a/data/linux/reverse_shell.yaml b/data/linux/reverse_shell.yaml new file mode 100644 index 0000000000..c0d036db99 --- /dev/null +++ b/data/linux/reverse_shell.yaml @@ -0,0 +1,13 @@ +title: Possible reverse shell command +description: Commands that look like reverse shell invocations +references: + - https://alamot.github.io/reverse_shells/ +logsource: + service: shell +detection: + keywords: + - '-i >& /dev/tcp/' + - 'exec 5<>/dev/tcp/' + - 'nc -e /bin/sh' + - "socat exec:'bash -li',pty,stderr,setsid,sigint,sane" +condition: keywords \ No newline at end of file diff --git a/data/sigma_config.yaml b/data/sigma_config.yaml new file mode 100644 index 0000000000..692e59c8c2 --- /dev/null +++ b/data/sigma_config.yaml @@ -0,0 +1,30 @@ +title: Timesketch Sigma config +order: 20 +backends: + - es-dsl + - es-qs +logsources: + sshd: + service: sshd + conditions: + data_type: "syslog/sshd" + auth: + service: auth + conditions: + data_type: "syslog" + apache: + product: apache + conditions: + data_type: "apache:access" + vsftp: + service: vsftp + conditions: + data_type: "vsftpd:log" + webserver: + category: webserver + conditions: + data_type: "apache:access OR iis:log:line" + shell: + service: shell + conditions: + data_type: "shell:zsh:history OR bash:history:command" \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 20df635422..1bf6f95712 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get -y install plaso-tools nodejs yarn RUN pip3 install --upgrade pip ADD . /tmp/timesketch RUN cd /tmp/timesketch && yarn install && yarn run build -# Remove pyyaml from requirements.txt to avoid conflits with python-yaml Ubuntu package +# Remove pyyaml from requirements.txt to avoid conflicts with python-yaml Ubuntu package RUN sed -i -e '/pyyaml/d' /tmp/timesketch/requirements.txt RUN pip3 install /tmp/timesketch/ @@ -38,6 +38,8 @@ RUN pip3 install /tmp/timesketch/ RUN mkdir /etc/timesketch RUN cp /tmp/timesketch/data/timesketch.conf /etc/timesketch/ RUN cp /tmp/timesketch/data/features.yaml /etc/timesketch/ +RUN cp /tmp/timesketch/data/sigma_config.yaml /etc/timesketch/ + # Copy the entrypoint script into the container COPY docker/docker-entrypoint.sh / diff --git a/requirements.txt b/requirements.txt index 3cfd417a13..9f4efe5cef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,8 +24,9 @@ python_dateutil==2.8.1 PyYAML==5.3 redis==3.3.11 requests==2.21.0 +sigmatools==0.14 ; python_version > '3.4' six==1.12.0 SQLAlchemy==1.3.12 Werkzeug==0.16.0 WTForms==2.2.1 -xlrd==1.2.0 \ No newline at end of file +xlrd==1.2.0 diff --git a/setup.py b/setup.py index 1317d4bf62..f41626b067 100755 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ ], data_files=[ ('share/timesketch', glob.glob( - os.path.join('data', '*'))), + os.path.join('data', '*'), recursive=True)), ('share/doc/timesketch', [ 'AUTHORS', 'LICENSE', 'README.md']), ], diff --git a/timesketch/lib/analyzers/__init__.py b/timesketch/lib/analyzers/__init__.py index 94819866aa..cea5ceaa0d 100644 --- a/timesketch/lib/analyzers/__init__.py +++ b/timesketch/lib/analyzers/__init__.py @@ -24,6 +24,7 @@ from timesketch.lib.analyzers import login from timesketch.lib.analyzers import phishy_domains from timesketch.lib.analyzers import sessionizer +from timesketch.lib.analyzers import sigma_tagger from timesketch.lib.analyzers import similarity_scorer from timesketch.lib.analyzers import ssh_sessionizer from timesketch.lib.analyzers import gcp_servicekey diff --git a/timesketch/lib/analyzers/interface.py b/timesketch/lib/analyzers/interface.py index 74be1b6047..eabd85a911 100644 --- a/timesketch/lib/analyzers/interface.py +++ b/timesketch/lib/analyzers/interface.py @@ -43,6 +43,27 @@ def wrapper(self, *args, **kwargs): return func_return return wrapper +def get_config_path(file_name): + """Returns a path to a configuration file. + + Args: + file_name: String that defines the config file name. + + Returns: + The path to the configuration file or None if the file cannot be found. + """ + path = os.path.join(os.path.sep, 'etc', 'timesketch', file_name) + if os.path.isfile(path): + return path + + path = os.path.join( + os.path.dirname(__file__), '..', '..', '..', 'data', file_name) + path = os.path.abspath(path) + if os.path.isfile(path): + return path + + return None + def get_yaml_config(file_name): """Return a dict parsed from a YAML file within the config directory. @@ -55,12 +76,8 @@ def get_yaml_config(file_name): an empty dict if the file is not found or YAML was unable to parse it. """ - root_path = os.path.join(os.path.sep, 'etc', 'timesketch') - if not os.path.isdir(root_path): - return {} - - path = os.path.join(root_path, file_name) - if not os.path.isfile(path): + path = get_config_path(file_name) + if not path: return {} with open(path, 'r') as fh: diff --git a/timesketch/lib/analyzers/sigma_tagger.py b/timesketch/lib/analyzers/sigma_tagger.py new file mode 100644 index 0000000000..abc8201d9d --- /dev/null +++ b/timesketch/lib/analyzers/sigma_tagger.py @@ -0,0 +1,116 @@ +"""Index analyzer plugin for sigma.""" +from __future__ import unicode_literals + +import logging +import os + +from sigma.backends import elasticsearch as sigma_elasticsearch +import sigma.configuration as sigma_configuration +from sigma.parser import collection as sigma_collection + + +from timesketch.lib.analyzers import interface +from timesketch.lib.analyzers import manager + + +class SigmaPlugin(interface.BaseSketchAnalyzer): + """Index analyzer for Sigma.""" + + NAME = 'sigma' + + _CONFIG_FILE = 'sigma_config.yaml' + + # Path to the directory containing the Sigma Rules to run, relative to + # this file. + _RULES_PATH = '' + + + def __init__(self, index_name, sketch_id): + """Initialize the Index Analyzer. + + Args: + index_name: Elasticsearch index name. + sketch_id: Sketch ID. + """ + super(SigmaPlugin, self).__init__(index_name, sketch_id) + sigma_config_path = interface.get_config_path(self._CONFIG_FILE) + logging.debug('[sigma] Loading config from {0!s}'.format( + sigma_config_path)) + with open(sigma_config_path, 'r') as sigma_config_file: + sigma_config = sigma_config_file.read() + self.sigma_config = sigma_configuration.SigmaConfiguration(sigma_config) + + def run_sigma_rule(self, query, tag_name): + """Runs a sigma rule and applies the appropriate tags. + + Args: + query: elastic search query for events to tag. + tag_name: tag to apply to matching events. + + Returns: + int: number of events tagged. + """ + return_fields = [] + tagged_events = 0 + events = self.event_stream( + query_string=query, return_fields=return_fields) + for event in events: + event.add_tags([tag_name]) + event.commit() + tagged_events += 1 + return tagged_events + + def run(self): + """Entry point for the analyzer. + + Returns: + String with summary of the analyzer result. + """ + sigma_backend = sigma_elasticsearch.ElasticsearchQuerystringBackend( + self.sigma_config, {}) + tags_applied = {} + + rules_path = os.path.join(os.path.dirname(__file__), self._RULES_PATH) + for rule_filename in os.listdir(rules_path): + tag_name, _ = rule_filename.rsplit('.') + tags_applied[tag_name] = 0 + rule_file_path = os.path.join(rules_path, rule_filename) + rule_file_path = os.path.abspath(rule_file_path) + logging.info('[sigma] Reading rules from {0!s}'.format( + rule_file_path)) + with open(rule_file_path, 'r') as rule_file: + rule_file_content = rule_file.read() + parser = sigma_collection.SigmaCollectionParser( + rule_file_content, self.sigma_config, None) + try: + results = parser.generate(sigma_backend) + except NotImplementedError as exception: + logging.error( + 'Error generating rule in file {0:s}: {1!s}'.format( + rule_file_path, exception)) + continue + + for result in results: + logging.info( + '[sigma] Generated query {0:s}'.format(result)) + number_of_tagged_events = self.run_sigma_rule( + result, tag_name) + tags_applied[tag_name] += number_of_tagged_events + + total_tagged_events = sum(tags_applied.values()) + output_string = 'Applied {0:d} tags\n'.format(total_tagged_events) + for tag_name, number_of_tagged_events in tags_applied.items(): + output_string += '* {0:s}: {1:d}'.format( + tag_name, number_of_tagged_events) + return output_string + + +class LinuxRulesSigmaPlugin(SigmaPlugin): + """Sigma plugin to run Linux rules.""" + + _RULES_PATH = '../../../data/linux' + + NAME = 'sigma_linux' + + +manager.AnalysisManager.register_analyzer(LinuxRulesSigmaPlugin) diff --git a/timesketch/lib/analyzers/sigma_tagger_test.py b/timesketch/lib/analyzers/sigma_tagger_test.py new file mode 100644 index 0000000000..4a060a5fe1 --- /dev/null +++ b/timesketch/lib/analyzers/sigma_tagger_test.py @@ -0,0 +1,28 @@ +"""Tests for SigmaPlugin.""" +from __future__ import unicode_literals + +import mock + +from timesketch.lib.analyzers import sigma_tagger +from timesketch.lib.testlib import BaseTest +from timesketch.lib.testlib import MockDataStore + + +class TestSigmaPlugin(BaseTest): + """Tests the functionality of the analyzer.""" + + def __init__(self, *args, **kwargs): + super(TestSigmaPlugin, self).__init__(*args, **kwargs) + self.test_index = 'test_index' + + + # Mock the Elasticsearch datastore. + @mock.patch( + 'timesketch.lib.analyzers.interface.ElasticsearchDataStore', + MockDataStore) + def test_analyzer(self): + """Test analyzer.""" + # TODO: Add more tests + + _ = sigma_tagger.LinuxRulesSigmaPlugin( + sketch_id=1, index_name=self.test_index)