Merge pull request #1 from jay-johnson/ssh-and-telnet

SSH and Telnet Capture Tools

README.rst

@@ -76,12 +76,16 @@ Here are the included, standalone capture tools (all of which require root privi
 
 #.  `capture_arp.py`_
 #.  `capture_icmp.py`_
+#.  `capture_ssh.py`_
 #.  `capture_tcp.py`_
+#.  `capture_telnet.py`_
 #.  `capture_udp.py`_
 
 .. _capture_arp.py: https://github.com/jay-johnson/network-pipeline/blob/master/network_pipeline/scripts/capture_arp.py
 .. _capture_icmp.py: https://github.com/jay-johnson/network-pipeline/blob/master/network_pipeline/scripts/capture_icmp.py
+.. _capture_ssh.py: https://github.com/jay-johnson/network-pipeline/blob/master/network_pipeline/scripts/capture_ssh.py
 .. _capture_tcp.py: https://github.com/jay-johnson/network-pipeline/blob/master/network_pipeline/scripts/capture_tcp.py
+.. _capture_telnet.py: https://github.com/jay-johnson/network-pipeline/blob/master/network_pipeline/scripts/capture_telnet.py
 .. _capture_udp.py: https://github.com/jay-johnson/network-pipeline/blob/master/network_pipeline/scripts/capture_udp.py
 
 AntiNex Stack Status
@@ -372,6 +376,18 @@ Scapy_ currently provides the traffic capture tooling, but the code already has
     
         ./network_pipeline/scripts/capture_tcp.py
 
+    Capture SSH Traffic
+
+    ::
+
+        ./network_pipeline/scripts/capture_ssh.py
+
+    Capture Telnet Traffic
+
+    ::
+
+        ./network_pipeline/scripts/capture_telnet.py
+
 #.  Capture UDP Data
 
     With another terminal, you can capture UDP traffic at the same time

docs/source/scripts.rst

@@ -1,22 +1,41 @@
+========================
 Network Pipeline Scripts
 ========================
 
 Capture Agents
---------------
+==============
 
 Here are the AntiNex Network Pipeline Capture Agents. These scripts allow for capturing traffic on a network device and flattening it into JSON dictionaries before publishing to the aggregation message broker. Please refer to the ``handle_packets`` method for more details.
 
 .. warning:: These tools will capture network traffic. Please be careful where you deploy them.
 
+ARP
+---
+
 .. automodule:: network_pipeline.scripts.capture_arp
    :members: capture_arp_packets
 
+ICMP
+----
+
 .. automodule:: network_pipeline.scripts.capture_icmp
    :members: capture_icmp_packets
 
+TCP
+---
+
+.. automodule:: network_pipeline.scripts.capture_ssh
+   :members: capture_tcp_packets_over_ssh
+
 .. automodule:: network_pipeline.scripts.capture_tcp
    :members: capture_tcp_packets
 
+.. automodule:: network_pipeline.scripts.capture_telnet
+   :members: capture_tcp_packets_over_telnet
+
+UDP
+---
+
 .. automodule:: network_pipeline.scripts.capture_udp
    :members: capture_udp_packets
 

network_pipeline/log/capture-arp-logging.json

@@ -0,0 +1,38 @@
+{
+    "version": 1,
+    "disable_existing_loggers": false,
+    "formatters": {
+        "simple": {
+            "()": "colorlog.ColoredFormatter",
+            "format": "%(log_color)s%(asctime)s - %(name)s - %(levelname)s - %(message)s%(reset)s"
+        }
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "level": "INFO",
+            "formatter": "simple",
+            "stream": "ext://sys.stdout"
+        },
+        "info_file_handler": {
+            "class": "logging.handlers.RotatingFileHandler",
+            "level": "INFO",
+            "formatter": "simple",
+            "filename": "capture-arp.log",
+            "maxBytes": 1048576,
+            "backupCount": 1,
+            "encoding": "utf8"
+        }
+    },
+    "loggers": {
+        "my_module": {
+            "level": "ERROR",
+            "handlers": ["console"],
+            "propagate": "no"
+        }
+    },
+    "root": {
+        "level": "INFO",
+        "handlers": ["console", "info_file_handler"]
+    }
+}

network_pipeline/log/capture-icmp-logging.json

@@ -0,0 +1,38 @@
+{
+    "version": 1,
+    "disable_existing_loggers": false,
+    "formatters": {
+        "simple": {
+            "()": "colorlog.ColoredFormatter",
+            "format": "%(log_color)s%(asctime)s - %(name)s - %(levelname)s - %(message)s%(reset)s"
+        }
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "level": "INFO",
+            "formatter": "simple",
+            "stream": "ext://sys.stdout"
+        },
+        "info_file_handler": {
+            "class": "logging.handlers.RotatingFileHandler",
+            "level": "INFO",
+            "formatter": "simple",
+            "filename": "capture-icmp.log",
+            "maxBytes": 1048576,
+            "backupCount": 1,
+            "encoding": "utf8"
+        }
+    },
+    "loggers": {
+        "my_module": {
+            "level": "ERROR",
+            "handlers": ["console"],
+            "propagate": "no"
+        }
+    },
+    "root": {
+        "level": "INFO",
+        "handlers": ["console", "info_file_handler"]
+    }
+}

network_pipeline/log/capture-ssh-logging.json

@@ -0,0 +1,38 @@
+{
+    "version": 1,
+    "disable_existing_loggers": false,
+    "formatters": {
+        "simple": {
+            "()": "colorlog.ColoredFormatter",
+            "format": "%(log_color)s%(asctime)s - %(name)s - %(levelname)s - %(message)s%(reset)s"
+        }
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "level": "INFO",
+            "formatter": "simple",
+            "stream": "ext://sys.stdout"
+        },
+        "info_file_handler": {
+            "class": "logging.handlers.RotatingFileHandler",
+            "level": "INFO",
+            "formatter": "simple",
+            "filename": "capture-ssh.log",
+            "maxBytes": 1048576,
+            "backupCount": 1,
+            "encoding": "utf8"
+        }
+    },
+    "loggers": {
+        "my_module": {
+            "level": "ERROR",
+            "handlers": ["console"],
+            "propagate": "no"
+        }
+    },
+    "root": {
+        "level": "INFO",
+        "handlers": ["console", "info_file_handler"]
+    }
+}

network_pipeline/log/capture-tcp-logging.json

@@ -0,0 +1,38 @@
+{
+    "version": 1,
+    "disable_existing_loggers": false,
+    "formatters": {
+        "simple": {
+            "()": "colorlog.ColoredFormatter",
+            "format": "%(log_color)s%(asctime)s - %(name)s - %(levelname)s - %(message)s%(reset)s"
+        }
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "level": "INFO",
+            "formatter": "simple",
+            "stream": "ext://sys.stdout"
+        },
+        "info_file_handler": {
+            "class": "logging.handlers.RotatingFileHandler",
+            "level": "INFO",
+            "formatter": "simple",
+            "filename": "capture-tcp.log",
+            "maxBytes": 1048576,
+            "backupCount": 1,
+            "encoding": "utf8"
+        }
+    },
+    "loggers": {
+        "my_module": {
+            "level": "ERROR",
+            "handlers": ["console"],
+            "propagate": "no"
+        }
+    },
+    "root": {
+        "level": "INFO",
+        "handlers": ["console", "info_file_handler"]
+    }
+}

network_pipeline/log/capture-telnet-logging.json

@@ -0,0 +1,38 @@
+{
+    "version": 1,
+    "disable_existing_loggers": false,
+    "formatters": {
+        "simple": {
+            "()": "colorlog.ColoredFormatter",
+            "format": "%(log_color)s%(asctime)s - %(name)s - %(levelname)s - %(message)s%(reset)s"
+        }
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "level": "INFO",
+            "formatter": "simple",
+            "stream": "ext://sys.stdout"
+        },
+        "info_file_handler": {
+            "class": "logging.handlers.RotatingFileHandler",
+            "level": "INFO",
+            "formatter": "simple",
+            "filename": "capture-telnet.log",
+            "maxBytes": 1048576,
+            "backupCount": 1,
+            "encoding": "utf8"
+        }
+    },
+    "loggers": {
+        "my_module": {
+            "level": "ERROR",
+            "handlers": ["console"],
+            "propagate": "no"
+        }
+    },
+    "root": {
+        "level": "INFO",
+        "handlers": ["console", "info_file_handler"]
+    }
+}

network_pipeline/log/capture-udp-logging.json

@@ -0,0 +1,38 @@
+{
+    "version": 1,
+    "disable_existing_loggers": false,
+    "formatters": {
+        "simple": {
+            "()": "colorlog.ColoredFormatter",
+            "format": "%(log_color)s%(asctime)s - %(name)s - %(levelname)s - %(message)s%(reset)s"
+        }
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "level": "INFO",
+            "formatter": "simple",
+            "stream": "ext://sys.stdout"
+        },
+        "info_file_handler": {
+            "class": "logging.handlers.RotatingFileHandler",
+            "level": "INFO",
+            "formatter": "simple",
+            "filename": "capture-udp.log",
+            "maxBytes": 1048576,
+            "backupCount": 1,
+            "encoding": "utf8"
+        }
+    },
+    "loggers": {
+        "my_module": {
+            "level": "ERROR",
+            "handlers": ["console"],
+            "propagate": "no"
+        }
+    },
+    "root": {
+        "level": "INFO",
+        "handlers": ["console", "info_file_handler"]
+    }
+}

network_pipeline/log/setup_logging.py

@@ -105,15 +105,23 @@ def build_colorized_logger(
     :param log_level: level to log
     :param log_config_path: path to log config file
     """
-    use_config = ("./log/{}").format(
-                    "{}".format(
-                        config))
+    use_config = ("{}").format(
+                    config)
     if not os.path.exists(use_config):
-        use_config = log_config_path
+        use_config = ("./network_pipeline/log/{}").format(
+                            config)
         if not os.path.exists(use_config):
-            use_config = ("./network_pipeline/log/{}").format(
-                            "logging.json")
-    # find the log processing
+            use_config = log_config_path
+            if not os.path.exists(use_config):
+                use_config = ("./log/{}").format(
+                            config)
+                if not os.path.exists(use_config):
+                    use_config = ("./network_pipeline/log/{}").format(
+                                "logging.json")
+                # find the last log config backup from the base of the repo
+            # find the log config from the defaults with the env LOG_CFG
+        # find the log config from the base of the repo
+    # find the log config by the given path
 
     setup_logging(
         default_level=log_level,

network_pipeline/record_packets_to_csv.py

@@ -1,11 +1,10 @@
 import os
 import sys
-import logging
 import json
 import pandas as pd
 from pandas.io.json import json_normalize
 from celery_connectors.utils import ev
-from network_pipeline.log.setup_logging import setup_logging
+from network_pipeline.log.setup_logging import build_colorized_logger
 from network_pipeline.utils import ppj
 from network_pipeline.utils import rnow
 from network_pipeline.build_packet_key import build_packet_key
@@ -23,9 +22,9 @@
 from antinex_client.generate_ai_request import generate_ai_request
 
 
-setup_logging()
 name = "csv"
-log = logging.getLogger(name)
+log = build_colorized_logger(
+        name=name)
 
 
 class RecordPacketsToCSV:
@@ -120,8 +119,9 @@ def __init__(self):
         # noqa https://github.com/jay-johnson/antinex-client/blob/master/examples/example-prediction.env
         self.request_dict = {}
         if ANTINEX_PUBLISH_ENABLED:
-            with open(ANTINEX_PUBLISH_REQUEST_FILE, "r") as f:
-                self.request_dict = json.loads(f.read())
+            if os.path.exists(ANTINEX_PUBLISH_REQUEST_FILE):
+                with open(ANTINEX_PUBLISH_REQUEST_FILE, "r") as f:
+                    self.request_dict = json.loads(f.read())
         # if publishing is enabled
 
     # end of __init__