initial version

README.md

@@ -0,0 +1,3 @@
+An easy ATT&CK-based Sysmon hunting tool
+===
+

agent.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+import argparse
+
+from utils.common import *
+from data import sysmon
+from db import es, graph
+from core import attck
+from core import rule
+
+def process_csv(conf, csv_file):
+    behavs = sysmon.SysmonData().from_csv(csv_file)
+    _es = es.ES(conf)
+    _es.insert_behaviors('raw', behavs)
+    attck_techs = attck.load_attcks(conf['attck_yaml'])
+
+    abnormals = rule.filter_abnormal_behaviors(behavs, attck_techs)
+    _es.insert_behaviors('abnormal', abnormals)
+    _neo4j = graph.Neo4jGraph(conf)
+    _neo4j.update_behaviors(abnormals)
+
+def process_winlogbeat(conf, start, end):
+    _es = es.ES(conf)
+    behavs = sysmon.SysmonData().from_winlogbeat(_es, 'winlogbeat-*', start, end)
+    _es.insert_behaviors('raw', behavs)
+    attck_techs = attck.load_attcks(conf['attck_yaml'])
+
+    abnormals = rule.filter_abnormal_behaviors(behavs, attck_techs)
+    _es.insert_behaviors('abnormal', abnormals)
+    _neo4j = graph.Neo4jGraph(conf)
+    _neo4j.update_behaviors(abnormals)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c', help='conf file')
+    parser.add_argument('-t', help='choose csv or winlogbeat')
+    parser.add_argument('-i', help='csv file')
+    parser.add_argument('-start', help='start date from winlogbeat, like 2019-07-19')
+    parser.add_argument('-end', help='end date from winlogbeat, like 2019-07-19')
+    args = parser.parse_args()
+
+
+    if args.t not in ['csv', 'winlogbeat']:
+        import sys
+        parser.print_usage()
+        sys.exit(1)
+
+    conf = parse_conf(args.c)
+    if args.t == 'csv':
+        process_csv(conf, args.i)
+    else:
+        process_winlogbeat(conf, args.start, args.end)
+

analyst/statistic.py

@@ -0,0 +1,38 @@
+
+from utils.log import *
+
+def _df_columns_statistic(data, cols):
+    subdata = data.dropna(subset=cols, how='all')
+    gpdata = subdata.groupby(by=cols).size().sort_values(ascending=False)
+
+    # for debugging.
+    #print gpdata.describe()
+
+    return gpdata
+
+
+def st_output(gpdata):
+    result = []
+    for _value, _count in gpdata.iteritems():
+        if type(_value) is list or type(_value) is tuple:
+            _value = '|'.join(_value)
+            result.append([_value, _count])
+        else:
+            result.append([_value, _count])
+    return result
+
+def st_procchain(data):
+    result = _df_columns_statistic(data, ['parent.image', 'current.image'])
+    return result
+
+def st_network(data, cols):
+    result = _df_columns_statistic(data, cols)
+    return result
+
+def st_reg(data):
+    result = _df_columns_statistic(data, ['reg.path'])
+    return result
+
+def st_file(data):
+    result = _df_columns_statistic(data, ['file.path'])
+    return result

core/attck.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+import yaml
+
+AttckLevel = [
+    'ignore', 
+    'low', 
+    'medium', 
+    'high', 
+    'critical'
+]
+
+AttckPhase = [
+    'Initial Access', 
+    'Execution', 
+    'Persistence', 
+    'Privilege Escalation', 
+    'Defense Evasion',
+    'Credential Access',
+    'Discovery',
+    'Lateral Movement',
+    'Collection',
+    'Command and Control',
+    'Exfiltration',
+    'Impact',
+]
+
+class ATTCKTech(object):
+    def __init__(self, _id, _raw):
+        self.id = _id
+        self.name = _raw['name']
+        self.description = _raw['description']
+        self.level = _raw['level']
+        self.phase = _raw['phase']
+        self.conditions = _raw['query']
+
+    def __str__(self):
+        return '{}: {} ({})'.format(self.id, self.name, self.description)
+
+
+def load_attcks(yaml_path):
+    techs = {}
+
+    with open(yaml_path) as _yaml:
+        rules = yaml.load(_yaml)
+        for _id, e in rules.iteritems():
+            techs[_id] = ATTCKTech(_id, e)
+    return techs

core/behavior.py

@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+
+from core.entity import *
+
+
+class BaseBehavior(object):
+    CONTEXT = []
+    def __init__(self, _raw):
+        self.attck_ids = ''
+        self.date = None
+        self.relation = ''
+
+    def getname(self):
+        return self.__class__.__name__
+
+    def get_value(self):
+        pass
+
+    def get_attribute_names(self):
+        attrs = ['timestamp', 'relation', 'attckids', 'behaviortype', 'value']
+        for _attr in self.__class__.CONTEXT:
+            attrs.extend([_attr + '.' + prop for prop in self.__dict__[_attr].__class__.PROPS])
+        return attrs
+
+    def serialize(self):
+        serobj = {'timestamp': self.date, 'relation': self.relation, 'attckids': self.attck_ids, 'behaviortype': self.getname(), 'value': self.get_value()}
+        for _attr in self.__class__.CONTEXT:
+            _obj = self.__dict__[_attr]
+            serobj.update({_attr + '.' + prop: _obj[prop] for prop in _obj.__class__.PROPS})
+
+        return serobj
+
+    @staticmethod
+    def deserialize(_raw):
+        mappings = {
+            'ProcessBehavior': ProcessBehavior,
+            'NetworkBehavior': NetworkBehavior,
+            'FileBehavior': FileBehavior,
+            'RegistryBehavior': RegistryBehavior,
+        }
+
+        behav_data = {}
+        behav_data['datetime'] = _raw['timestamp']
+        behav_data['relation'] = _raw['relation']
+        cols = list(_raw.keys())
+        for col in cols:
+            if '.' in col:
+                if col.split('.')[0] not in behav_data.keys():
+                    behav_data[col.split('.')[0]] = {}
+                behav_data[col.split('.')[0]][col.split('.')[1]] = _raw[col]
+
+        _instance = mappings[_raw['behaviortype']](behav_data)
+        _instance.attck_ids = _raw['attckids']
+        return _instance
+
+
+class ProcessBehavior(BaseBehavior):
+    CONTEXT = ['parent', 'current', 'file', 'endpoint']
+    def __init__(self, _raw):
+        super(ProcessBehavior, self).__init__(_raw)
+
+        self.parent = ProcessEntity(_raw['parent'])
+        self.current = ProcessEntity(_raw['current'])
+        self.file = FileEntity(_raw['file'])
+        self.date = _raw['datetime']
+        self.endpoint = EndPointEntity(_raw['endpoint'])
+        self.relation = _raw['relation']
+
+    def __str__(self):
+        return '{} Endpoint({}) on {}: ({} {}) -{}-> ({} {})'.format(self.__class__.__name__, self.endpoint['uuid'], self.date, self.parent['image'], self.parent['cmdline'], self.relation, self.current['image'], self.current['cmdline'])
+
+    def get_value(self):
+        return '({} {}) -{}-> ({} {})'.format(self.parent['image'], self.parent['cmdline'], self.relation, self.current['image'], self.current['cmdline'])
+
+
+class NetworkBehavior(BaseBehavior):
+    CONTEXT = ['process', 'network', 'file', 'endpoint']
+    def __init__(self, _raw):
+        super(NetworkBehavior, self).__init__(_raw)
+
+        self.process = ProcessEntity(_raw['process'])
+        self.network = NetworkEntity(_raw['network'])
+        self.file = FileEntity(_raw['file'])
+        self.date = _raw['datetime']
+        self.endpoint = EndPointEntity(_raw['endpoint'])
+        self.relation = _raw['relation']
+
+    def __str__(self):
+        return '{} Endpoint({}) on {}: {} -{}-> {}({})'.format(self.__class__.__name__, self.endpoint['uuid'], self.date, self.process['image'], self.relation, self.network['rhost'], self.network['rip'])
+
+    def get_value(self):
+        return '{} -{}-> {}({})'.format(self.process['image'], self.relation, self.network['rhost'], self.network['rip'])
+
+class FileBehavior(BaseBehavior):
+    CONTEXT = ['process', 'file', 'endpoint']
+    def __init__(self, _raw):
+        super(FileBehavior, self).__init__(_raw)
+
+        self.process = ProcessEntity(_raw['process'])
+        self.file = FileEntity(_raw['file'])
+        self.date = _raw['datetime']
+        self.endpoint = EndPointEntity(_raw['endpoint'])
+        self.relation = _raw['relation']
+
+    def __str__(self):
+        return '{} Endpoint({}) on {}: {} -{}-> {}'.format(self.__class__.__name__, self.endpoint['uuid'], self.date, self.process['image'], self.relation, self.file['path'])
+
+    def get_value(self):
+        return '{} -{}-> {}'.format(self.process['image'], self.relation, self.file['path'])
+
+class RegistryBehavior(BaseBehavior):
+    CONTEXT = ['process', 'reg', 'file', 'endpoint']
+    def __init__(self, _raw):
+        super(RegistryBehavior, self).__init__(_raw)
+
+        self.process = ProcessEntity(_raw['process'])
+        self.reg = RegistryEntity(_raw['reg'])
+        self.file = FileEntity(_raw['file'])
+        self.date = _raw['datetime']
+        self.endpoint = EndPointEntity(_raw['endpoint'])
+        self.relation = _raw['relation']
+
+    def __str__(self):
+        return '{} Endpoint({}) on {}: {} -{}-> {} {} {}'.format(self.__class__.__name__, self.endpoint['uuid'], self.date, self.process['image'], self.relation, self.reg['path'], self.reg['key'], self.reg['value'])
+
+    def get_value(self):
+        return '{} -{}-> {} {} {}'.format(self.process['image'], self.relation, self.reg['path'], self.reg['key'], self.reg['value'])
+
+
+BEHAVIOR_SETS = [
+    ProcessBehavior,
+    NetworkBehavior,
+    FileBehavior,
+    RegistryBehavior,
+]

core/entity.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+import pandas
+from core.utils import *
+
+class BaseEntity(object):
+    TYPE = 'base'
+    PROPS = []
+    def __init__(self, _raw):
+        self.props = {}
+
+        if _raw:
+            for key, value in _raw.iteritems():
+                self.__setitem__(key, value)
+
+    def __getitem__(self, key):
+        if key not in self.__class__.PROPS:
+            raise KeyError()
+
+        return self.props[key] if key in self.props.keys() else ''
+
+    def __setitem__(self, key, value):
+        if key not in self.__class__.PROPS:
+            raise KeyError()
+
+        if value is None or value is pandas.np.nan:
+            self.props[key] = ''
+        else:
+            self.props[key] = value.encode('utf-8')
+
+
+class FileEntity(BaseEntity):
+    TYPE = 'file'
+    PROPS = ['hash', 'path', 'name', 'sig', 'type']
+
+    def __init__(self, _raw):
+        super(FileEntity, self).__init__(_raw)
+
+class ProcessEntity(BaseEntity):
+    TYPE = 'process'
+    PROPS = ['pid', 'image', 'cmdline', 'user', 'calltrace', 'guid']
+
+    def __init__(self, _raw):
+        super(ProcessEntity, self).__init__(_raw)
+
+class NetworkEntity(BaseEntity):
+    TYPE = 'network'
+    PROPS = ['clientip', 'clientport', 'rip', 'rport', 'protocol', 'rhost', 'ua', 'url']
+
+    def __init__(self, _raw):
+        super(NetworkEntity, self).__init__(_raw)
+
+class RegistryEntity(BaseEntity):
+    TYPE = 'reg'
+    PROPS = ['path', 'key', 'value']
+
+    def __init__(self, _raw):
+        super(RegistryEntity, self).__init__(_raw)
+
+
+class EndPointEntity(BaseEntity):
+    TYPE = 'endpoint'
+    PROPS = ['uuid', 'ip']
+
+    def __init__(self, _raw):
+        super(EndPointEntity, self).__init__(_raw)
+
+class UserEntity(BaseEntity):
+    TYPE = 'user'
+    PROPS = ['name', 'domain', 'privilege']
+
+    def __init__(self, _raw):
+        super(UserEntity, self).__init__(_raw)
+
+class ServiceEntity(BaseEntity):
+    pass