deleted: constat.py

	modified:   dbmodels.py
	deleted:    self_ioc_list.csv
	modified:   src/constat.py
	new file:   src/csv_load_to_db.py
	renamed:    ioc_self_get.py -> src/ioc_self_get.py
	renamed:    ioc_self_watch.py -> src/ioc_self_watch.py
	renamed:    iocwatch.py -> src/iocwatch.py
	new file:   src/self_ioc_list.csv

dbmodels.py

@@ -4,6 +4,7 @@
 from infi.clickhouse_orm.fields import *
 from infi.clickhouse_orm.engines import MergeTree
 from infi.clickhouse_orm.engines import Memory
+from infi.clickhouse_orm.engines import Buffer
 
 
 class OPEN_PORTS(Model):
@@ -90,6 +91,7 @@ class DISKStats(Model):
 
     engine = MergeTree('event_date', ('total', 'used', 'timestamp'))
 
+
 class CONNStats(Model):
     event_date = DateField()
     timestamp = DateTimeField()
@@ -114,4 +116,4 @@ class CONNStats_buffer(Model):
     dst_port = Float32Field()
     qry_name = StringField()
 
-    ENGINE = Buffer('connstats', 16, 10, 100, 10000, 1000000, 10000000, 100000000)
+    ENGINE = Buffer('connstats', 16, 10, 100, 10000, 1000000, 10000000, 100000000)

src/constat.py

@@ -7,8 +7,10 @@
 import datetime
 #import os
 #import psutil
-#import time
+import time
 import logging
+import requests
+import csv
 
 import pyshark
 import pytz
@@ -17,13 +19,14 @@
 import dbmodels
 
 # Set logging level
-logging.basicConfig(level = logging.INFO)
+logging.basicConfig(level = logging.ERROR)
 
 
 # Read config
 with open("/etc/politraf/config.yaml", 'r') as stream:
     try:
         config = (yaml.safe_load(stream))
+        TRAF_FILE = config['ioc_file']
         interface = config['interface']
         interfaces = interface.split(",")
         bpf_filter = config['bpf_filter']
@@ -54,41 +57,85 @@
 def database_write(today, timestamp, protocol, src_addr, src_port, dst_addr, dst_port, qry_name):
     try:
         db.insert([
-            dbmodels.CONNStats(event_date=today, timestamp=timestamp, protocol=protocol, src_addr=src_addr, src_port=src_port, dst_addr=dst_addr, dst_port=dst_port, qry_name=qry_name)
+            dbmodels.CONNStats_buffer(event_date=today, timestamp=timestamp, protocol=protocol, src_addr=src_addr, src_port=src_port, dst_addr=dst_addr, dst_port=dst_port, qry_name=qry_name)
             ])
     except Exception as e:
         logging.error(e)
 
+def database_write_bulk(data_buffer):
+    try:
+        start = time.time()
+        db.insert(data_buffer)
+        end = time.time()
+        time_to_end = end - start
+        print('Time to send all events ', time_to_end)
+    except Exception as e:
+        logging.error(e)
+
+def csv_file_write(today, timestamp, protocol, src_addr, src_port, dst_addr, dst_port, qry_name):
+    today=str(today)
+    timestamp=str(timestamp)
+    protocol=str(protocol)
+    src_addr=str(src_addr)
+    src_port=str(src_port)
+    dst_addr=str(dst_addr)
+    dst_port=str(dst_port)
+    qry_name=str(qry_name)
+    with open('/opt/politraf/current/traff.csv', 'a') as csvfile:
+        spamwriter = csv.writer(csvfile, delimiter=',',
+                                quotechar='|', quoting=csv.QUOTE_MINIMAL)
+        spamwriter.writerow([today, timestamp, protocol, src_addr, src_port, dst_addr, dst_port, qry_name])
 
 def print_conversation_header(pkt):
     try:
+        start = time.time()
         protocol = pkt.transport_layer
-        if protocol == "UDP" or protocol == "TCP":
-            src_addr = pkt.ip.src
-            src_port = pkt[pkt.transport_layer].srcport
-            dst_addr = pkt.ip.dst
-            dst_port = pkt[pkt.transport_layer].dstport
-
-            # UDP traf
-            if protocol == "UDP":
-                # DNS request
-                if "dns" in pkt:
-                    if pkt.dns.qry_name:
-                        qry_name = pkt.dns.qry_name
-                else:
-                    qry_name = "none"
-
-            # TCP traf
-            if protocol == "TCP":
-                if "http.host" in pkt:
-                    qry_name = pkt.http.host
-                else:
-                    qry_name = "none"
-            local = pytz.timezone(tz)
-            timestamp = local.localize(datetime.datetime.now())
-            today = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d')
-
-            database_write(today, timestamp, protocol, src_addr, src_port, dst_addr, dst_port, qry_name)
+
+        src_addr = pkt.ip.src
+        src_port = pkt[pkt.transport_layer].srcport
+        dst_addr = pkt.ip.dst
+        dst_port = pkt[pkt.transport_layer].dstport
+        qry_name = "none"
+
+        # UDP traf
+        if protocol == "UDP":
+        # DNS request
+            if "dns" in pkt:
+                if pkt.dns.qry_name:
+                    qry_name = pkt.dns.qry_name
+        # TCP traf
+        if protocol == "TCP":
+            if "http.host" in pkt:
+                qry_name = pkt.http.host
+
+        local = pytz.timezone(tz)
+        timestamp = local.localize(datetime.datetime.now())
+        today = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d')
+
+        # prepare data for bulk
+        data_buffer = []
+        insert_data = dbmodels.CONNStats_buffer(
+            event_date=today,
+            timestamp=timestamp,
+            protocol=protocol,
+            src_addr=src_addr,
+            src_port=src_port,
+            dst_addr=dst_addr,
+            dst_port=dst_port,
+            qry_name=qry_name
+            )
+        # appends data into couple
+        #data_buffer.append(insert_data)
+        #database_write_bulk(data_buffer)
+        csv_file_write(today, timestamp, protocol, src_addr, src_port, dst_addr, dst_port, qry_name)
+        #print(today, timestamp, protocol, src_addr, src_port, dst_addr, dst_port, qry_name)
+        #print(str(insert_data))
+        end = time.time()
+        time_to_end = end - start
+        #print('Time to analyze packet ', time_to_end)
+
+        #database_write_bulk(data_buffer)
+        #print(data_buffer, insert_data)
 
     except Exception as e:
         logging.error(e)
@@ -98,6 +145,13 @@ def main():
     try:
         logging.info("Running tshark with: interface: "+interface+" and bpf_filter: "+bpf_filter+ " and send connections stats to clickhouse")
         cap.apply_on_packets(print_conversation_header)
+        #cap.sniff(timeout=1)
+        #var = 1
+        #while var == 1:
+        #    for pkt in cap.sniff_continuously(packet_count=4):
+        #        print_conversation_header(pkt)
+        #        cap.close()
+
     except Exception as e:
         logging.error(e)
 if __name__ == '__main__':