Merge pull request #1 from cruikshank25/generalised_log_generator

Generalised log generator into main.

.gitignore

@@ -1,5 +1,9 @@
 # ignore pycache
 __pycache__/
 
-# ignore log
-security.log
+# ignore logs
+ids.log
+access.log
+
+# ignore tests directory
+tests/

README.md

@@ -1,18 +1,47 @@
 # Security Log Generator
-
-## Description 
-
-- Generates simulated security log events of similar format to IDS systems like Snort.
-
-- View the 'security_log_sample.log' for example log events that are produced. 
-
-## How To Use
-
+![PyPI - Python Version](https://img.shields.io/pypi/pyversions/Faker) [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
+
+## Features
+- Generates simulated log events for log formats that would typically be found in a SIEM solution.
+- As of 11th January 2023, it supports IDS and Web access log formats.
+- In the future, the following log formats are planned to be added:
+    - Windows event logs
+    - Linux event logs
+    - Perimeter device logs (firewalls, vpns, proxies etc)
+    - Endpoint security logs
+    - Feel free to add an issue to the project if you require an additional format not listed
+- The code is modular and extensible, adding additional formats can be done with relative ease. 
+
+## Architecture
+![My Image](Security_Log_Generator_Architecture.png)
+
+## Requirements
+- A Python installation above Version 3.7+
+- install libraries with pip: 
+(currently the only dependency is Faker - https://pypi.org/project/Faker/)
+~~~
+pip install -r requirements.txt
+~~~
+
+## Configuration
+- There are three main configuration parameters:
+    - no_events : this defines the number of events to produce
+    - write_time : this defines how long it takes to write an event (in milliseconds)
+    - log_type : this defines what type of log to produce
+- You edit these values directly in the config.py file.
+- An example valid configuration could look like this:
+~~~
+config = {
+    "logging_level":"INFO",
+    "no_events":500,
+    "write_time":0.25,
+    "log_type":"ids",
+}
+~~~
+- for 'log_type', the current valid values are 'ids' and 'access'. 
+
+## Run
 - To run, simply run 'python main.py' in the terminal.
-
-- To change the number of log events to be generated, use the 'no_events' setting in the config.py file.
-
-- To change the time taken to write an event, modify the 'write_time' setting in the config.py file.
-The lower the write time, the faster your event's will be generated (default is 0.25 (250 ms)).
-
-- Tested with Python version 3.10.6 (should be compatible with any version above 3.6+).
+~~~
+python main.py
+~~~

config.py

@@ -1,9 +1,11 @@
 config = {
     "logging_level":"INFO",
     "no_events":500,
-    "write_time":0.25
+    "write_time":0.25,
+    "log_type":"ids",
 }
 
 logging_level = config["logging_level"]
 no_events = int(config["no_events"])
-write_time = float(config["write_time"])
+write_time = float(config["write_time"])
+log_type = config["log_type"]

events.py

@@ -0,0 +1,23 @@
+# IDS event class representing the contents of an IDS security log event
+class ids_event:
+    def __init__(self, severity, protocol, src_ip, dest_ip, src_port, dest_port, flags, alert_desc):
+        self.severity = severity
+        self.protocol = protocol
+        self.src_ip = src_ip
+        self.dest_ip = dest_ip
+        self.src_port = src_port
+        self.dest_port = dest_port
+        self.flags = flags
+        self.alert_desc = alert_desc
+
+
+class access_event:
+    def __init__(self, client_ip, method, resource, protocol, status, bytes, referrer, user_agent):
+        self.client_ip = client_ip
+        self.method = method
+        self.resource = resource
+        self.protocol = protocol
+        self.status = status
+        self.bytes = bytes
+        self.referrer = referrer
+        self.user_agent = user_agent

fields.py

@@ -0,0 +1,50 @@
+# IDS log possible fields and possible values for field
+class ids_fields:
+    def __init__(self):
+         # define possible field values and weights for field values (the liklihood of a particular field value being selected
+        self.PROTOCOL = ['TCP', 'UDP', 'HTTP', 'HTTPS', 'ICMP', 'FTP', 'SMTP', 'DNS', 'DHCP', 'TFTP', 'SNMP']
+        self.PROTOCOL_WEIGHTS = [10, 4, 4, 4, 3, 1, 1, 1, 1, 1, 1]
+
+        self.FLAG = ['SYN', 'ACK', 'FIN', 'RST', 'PSH', 'URG']
+        self.FLAG_WEIGHTS = [10, 6, 4, 2, 2, 1]
+
+        self.ALERT_DESCRIPTION = ['Port scanning', 'Denial of service (DoS)', 'PING NMAP', 'Malicious traffic',
+            'Phishing', 'Malware', 'SQL Injection', 'Cross-site scripting (XSS)', 'Worm Propagation Attempt']
+        self.ALERT_WEIGHTS = [9, 8, 7, 6, 5, 4, 3, 2, 1]
+
+        self.SEVERITY = ['low_severity', 'medium_severity', 'high_severity', 'critical_severity']
+        self.SEVERITY_WEIGHTS = [20, 4, 2, 1]
+
+ids_fields = ids_fields()
+
+
+class access_fields:
+    def __init__(self):
+        self.METHOD = ['GET', 'POST', 'PUT', 'DELETE', 'HEAD', 'CONNECT', 'OPTIONS', 'TRACE', 'PATCH']
+        self.METHOD_WEIGHTS = [20, 6, 4, 2, 1, 1, 1, 1, 1]
+
+        self.PROTOCOL = ['HTTP/1.1', 'HTTPS/1.1', 'HTTPS/2.0']
+        self.PROTOCOL_WEIGHTS = [20, 10, 5]
+
+        self.STATUS = ['200', '201', '202', '203', '204', '205', '206',
+                        '300', '301', '302', '303', '304', '305', '307',
+                        '400', '401', '402', '403', '404', '405', '406',
+                        '407', '408', '409', '410', '411', '412', '413',
+                        '414', '415', '416', '417',
+                        '500', '501', '502', '503', '504', '505']
+        self.STATUS_WEIGHTS = [40, 5, 5, 5, 5, 5, 5,
+                               10, 2, 2, 2, 2, 2, 2,
+                               20, 2, 2, 2, 2, 2, 2,
+                               2, 2, 2, 2, 2, 2, 2,
+                               2, 2, 2, 2,
+                               10, 1, 1, 1, 1, 1]
+
+        self.USER_AGENT = ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36',
+                           'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15',
+                           'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
+                           'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36',
+                           '"Googlebot/2.1 (+http://www.googlebot.com/bot.html)"']
+        self.USER_AGENT_WEIGHTS = [5, 5, 5, 1, 1]
+
+
+access_fields = access_fields()

generators/access_generator.py

@@ -0,0 +1,50 @@
+import random 
+import ipaddress
+from events import access_event
+from fields import access_fields
+from faker import Faker
+
+def get_ip():
+    # generate a random octet (a number between 0 and 255)
+    octet1 = random.randint(0, 255)
+    octet2 = random.randint(0, 255)
+    octet3 = random.randint(0, 255)
+    octet4 = random.randint(0, 255)
+
+    # create a string representation of the ip address
+    ip_str = f"{octet1}.{octet2}.{octet3}.{octet4}"
+
+    # create an ip address object using the ip_address function
+    ip_addr = ipaddress.ip_address(ip_str)
+
+    return ip_addr
+
+
+def get_url():
+
+    faker = Faker()
+    domain_name = faker.domain_name()
+    url_path = faker.uri_path()
+    file_extension = faker.uri_extension()
+    param1 = faker.name()
+    param2 = faker.random_number()
+    param3 = faker.date_time_this_decade()
+    query_string = f'param1={param1}&param2={param2}&param3={param3}'
+    url = f"http://{domain_name}{url_path}{file_extension}?{query_string}"
+
+    return url_path, url
+
+
+def make_event():
+
+    event_client_ip = get_ip()
+    event_resource, event_url = get_url()
+    event_method = random.choices(access_fields.METHOD, access_fields.METHOD_WEIGHTS)[0]
+    event_protocol = random.choices(access_fields.PROTOCOL, access_fields.PROTOCOL_WEIGHTS)[0]
+    event_status = random.choices(access_fields.STATUS, access_fields.STATUS_WEIGHTS)[0]
+    event_bytes = random.randint(1, 100000)
+    event_user_agent = random.choices(access_fields.USER_AGENT, access_fields.USER_AGENT_WEIGHTS)[0]
+
+    event = access_event(event_client_ip, event_method, event_resource, event_protocol, event_status, event_bytes, event_url, event_user_agent)
+
+    return event

generators/ids_generator.py

@@ -0,0 +1,63 @@
+import random
+import ipaddress
+from fields import ids_fields
+from events import ids_event
+
+
+def get_port(protocol):
+    # create a dictionary of common port values for protocols
+    protocol_to_port = {
+        'TCP': random.randint(1, 65535),
+        'UDP': random.randint(1, 65535),
+        'ICMP': 1,
+        'HTTP': 80,
+        'HTTPS': 443,
+        'FTP': 21,
+        'SMTP': 25,
+        'DNS': 53,
+        'DHCP': 67,
+        'TFTP': 69,
+        'SNMP': 161
+    }
+
+    # return the port for the protocol, if not protocol from above, generate random port number
+    return protocol_to_port.get(protocol, random.randint(1, 65535))
+
+
+def get_ip():
+    # generate a random octet (a number between 0 and 255)
+    octet1 = random.randint(0, 255)
+    octet2 = random.randint(0, 255)
+    octet3 = random.randint(0, 255)
+    octet4 = random.randint(0, 255)
+
+    # create a string representation of the ip address
+    ip_str = f"{octet1}.{octet2}.{octet3}.{octet4}"
+
+    # create an ip address object using the ip_address function
+    ip_addr = ipaddress.ip_address(ip_str)
+
+    return ip_addr
+
+
+def make_event():
+    # create the severity, protocol, flag and alert description from the possible choices based on the weights
+    event_severity = random.choices(ids_fields.SEVERITY, ids_fields.SEVERITY_WEIGHTS)[0]
+    event_protocol = random.choices(ids_fields.PROTOCOL, ids_fields.PROTOCOL_WEIGHTS)[0]
+    event_flag = random.choices(ids_fields.FLAG, ids_fields.FLAG_WEIGHTS)[0]
+    event_alert_desc = random.choices(ids_fields.ALERT_DESCRIPTION, ids_fields.ALERT_WEIGHTS)[0]
+
+    # use the get_ip method to generate random valid ip addresses for source and destination
+    event_src_ip = get_ip()
+    event_dest_ip = get_ip()
+
+    # create a random valid source port
+    event_src_port = random.randint(1, 65535)
+
+    # create the dest port based on the protocol, if not then generate a random valid port
+    event_dest_port = event_dest_port = get_port(event_protocol)
+
+    # create the event using the 'Event' class and return the 'Event' object
+    event = ids_event(event_severity, event_protocol, event_src_ip, event_dest_ip, event_src_port, event_dest_port, event_flag, event_alert_desc)
+
+    return event