initial code automodules

docs/source/consts.rst

@@ -0,0 +1,74 @@
+Constants
+=========
+
+::
+	
+	VALID = 0
+	FILTERED = 1
+	INVALID = 2
+	ERROR = 3
+	UNSUPPORTED = 4
+	ETH_UNSUPPORTED = 5
+	IP_UNSUPPORTED = 6
+
+	INCLUDED_IGNORE_KEY = "CHANGE_TO_YOUR_OWN_KEY"
+
+	ETH_HEADER_FORMAT = "!6s6sH"
+	IP_HEADER_FORMAT = "!BBHHHBBH4s4s"
+	TCP_HEADER_FORMAT = "!HHLLBBHHH"
+	TCP_PSH_FORMAT = "!4s4sBBH"
+	UDP_HEADER_FORMAT = "!HHHH"
+	ICMP_HEADER_FORMAT = "!BBH"
+	ARP_HEADER_FORMAT = "2s2s1s1s2s6s4s6s4s"
+
+	SIZE_ETH_HEADER = struct.calcsize(ETH_HEADER_FORMAT)
+	SIZE_IP_HEADER = struct.calcsize(IP_HEADER_FORMAT)
+	SIZE_TCP_HEADER = struct.calcsize(TCP_HEADER_FORMAT)
+	SIZE_UDP_HEADER = struct.calcsize(UDP_HEADER_FORMAT)
+	SIZE_ICMP_HEADER = struct.calcsize(ICMP_HEADER_FORMAT)
+	SIZE_ARP_HEADER = struct.calcsize(ARP_HEADER_FORMAT)
+
+	UNKNOWN = 0
+	TCP = 1
+	UDP = 2
+	ICMP = 3
+	ARP = 4
+
+	ARP_PROTO_ETH = 9731
+	ICMP_PROTO_IP = 1
+	IP_PROTO_ETH = 8
+	TCP_PROTO_IP = 6
+	UDP_PROTO_IP = 17
+
+	IGNORED_REDIS_PORTS = [6379, 16379]
+	IGNORED_RABBITMQ_PORTS = [5672, 15672, 25672]
+
+Environment Variables
+=====================
+
+::
+
+	SOURCE = os.getenv(
+		"SOURCE_HOST",
+		"localdev").strip().lstrip()
+	FORWARD_BROKER_URL = os.getenv(
+		"FORWARD_BROKER_URL",
+		"redis://localhost:6379/0").strip().lstrip()
+	FORWARD_SSL_OPTIONS = json.loads(os.getenv(
+		"FORWARD_SSL_OPTIONS",
+		"{}").strip().lstrip())
+	FORWARD_ENDPOINT_TYPE = os.getenv(
+		"FORMAT_ET",
+		"redis").strip().strip()
+	FORWARD_EXCHANGE = os.getenv(
+		"FORWARD_EXCHANGE",
+		"NEW_PACKETS").strip().lstrip()
+	FORWARD_ROUTING_KEY = os.getenv(
+		"FORWARD_ROUTING_KEY",
+		"NEW_PACKETS").strip().lstrip()
+	FORWARD_QUEUE = os.getenv(
+		"FORWARD_QUEUE",
+		"NEW_PACKETS").strip().lstrip()
+	DEBUG_PACKETS = bool(os.getenv(
+		"DEBUG_PACKETS",
+		"0").strip().lstrip() == "1")

docs/source/index.rst

@@ -60,15 +60,21 @@ AntiNex Network Pipeline is part of the AntiNex stack:
            :target: https://readthedocs.org/projects/antinex-client/badge/?version=latest
 
 Table of Contents
------------------
+=================
 
 These are the docs for the AntiNex Network Pipeline repository.
 
+Source Code
+-----------
+
 .. toctree::
    :maxdepth: 2
 
    handle_packets
    record_packets_to_csv
+   internal_modules
+   scripts
+   consts
 
 Indices and tables
 ==================

docs/source/internal_modules.rst

@@ -0,0 +1,42 @@
+Network Pipeline Internal Modules
+=================================
+
+.. automodule:: network_pipeline.build_packet_key
+   :members: build_packet_key
+
+.. automodule:: network_pipeline.build_training_request
+   :members: build_training_request
+
+.. automodule:: network_pipeline.connect_forwarder
+   :members: connect_forwarder
+
+.. automodule:: network_pipeline.convert_pkt_to_json
+   :members: convert_pkt_to_json
+
+.. automodule:: network_pipeline.create_layer_2_socket
+   :members: create_layer_2_socket
+
+
+.. automodule:: network_pipeline.parse_network_data
+   :members: eth_addr,unshift_flags,build_key,parse_network_data
+
+.. automodule:: network_pipeline.publisher
+   :members: get_publisher
+
+.. automodule:: network_pipeline.utils
+   :members: rnow,ppj
+
+.. automodule:: network_pipeline.start_consumers_for_queue
+   :members: start_consumers_for_queue
+
+.. automodule:: network_pipeline.network_packet_task
+   :members: NetworkPacketTask
+
+.. automodule:: network_pipeline.shutdown_consumers
+   :members: shutdown_consumers
+
+.. automodule:: network_pipeline.simulated_work_task
+   :members: SimulatedWorkTask
+
+.. automodule:: network_pipeline.worker_to_process_packets.py
+   :members: WorkerToProcessPackets

docs/source/record_packets_to_csv.rst

@@ -1,5 +1,5 @@
-Source Code - Process Consumed Messages from the Queue
-======================================================
+Process Consumed Messages from the Queue
+========================================
 
 This is the default handler for processing messages consumed from the aggregration message broker. At the conceptual level, all network interface capture tools forward JSON dictionaries to this class.
 

docs/source/scripts.rst

@@ -0,0 +1,58 @@
+Network Pipeline Scripts
+========================
+
+Capture Agents
+--------------
+
+Here are the AntiNex Network Pipeline Capture Agents. These scripts allow for capturing traffic on a network device and flattening it into JSON dictionaries before publishing to the aggregation message broker. Please refer to the ``handle_packets`` method for more details.
+
+.. warning:: These tools will capture network traffic. Please be careful where you deploy them.
+
+.. automodule:: network_pipeline.scripts.capture-arp
+   :members: dev,default_filter,custom_filter,scapy
+
+.. automodule:: network_pipeline.scripts.capture-icmp
+   :members: dev,default_filter,custom_filter,scapy
+
+.. automodule:: network_pipeline.scripts.capture-tcp
+   :members: dev,default_filter,custom_filter,scapy
+
+.. automodule:: network_pipeline.scripts.capture-udp
+   :members: dev,default_filter,custom_filter,scapy
+
+Publishers
+----------
+
+These tools are designed to show how to save captured packet dictionaries to CSVs and how to publish them for live predictions using a pre-trained Deep Neural Network.
+
+.. automodule:: network_pipeline.scripts.packets-rabbitmq
+   :members: agg,recv_msg,FORWARD_BROKER_URL,FORWARD_SSL_OPTIONS,FORWARD_QUEUE,sub,queue,seconds
+
+.. automodule:: network_pipeline.scripts.packets-redis
+   :members: agg,recv_msg,FORWARD_BROKER_URL,FORWARD_SSL_OPTIONS,FORWARD_QUEUE,sub,queue,seconds
+
+Test Tools
+----------
+
+These will send mock traffic data to the targetd network device.
+
+.. automodule:: network_pipeline.scripts.arp-send-msg
+   :members: def,network_details,dst_ip,dst_msc,answered,unanswered
+
+.. automodule:: network_pipeline.scripts.icmp-send-msg
+   :members: main,checksum,do_one,send_one_ping,receive_one_ping,dump_stats,signal_handler,verbose_ping,quiet_ping
+
+.. automodule:: network_pipeline.scripts.tcp-send-large-msg
+   :members: client,msg
+
+.. automodule:: network_pipeline.scripts.tcp-send-msg
+   :members: client,msg
+
+.. automodule:: network_pipeline.scripts.udp-send-msg
+   :members: client,msg
+
+.. automodule:: network_pipeline.scripts.listen-tcp-port
+   :members: client,s
+
+.. automodule:: network_pipeline.scripts.listen-udp-port
+   :members: client,s