Develop 5.0 (#10)

* Plugin interface version 2
* Plugin CLI integration
* `--version`
* Warn and drop frame on invalid timestamp
* Remove dnslink plugin
* More test coverage

.coveragerc

@@ -0,0 +1,4 @@
+[run]
+omit = 
+    # omit plugin skeleton
+    pdml2flow/plugin-skeleton/*

.travis.yml

@@ -3,7 +3,10 @@ language: python
 python:
 - '3.4'
 - '3.5'
-- 3.5-dev
+- '3.5-dev'
+- '3.6'
+- '3.6-dev'
+- '3.7-dev'
 - nightly
 # VERSION END
 # Using trusty: https://docs.travis-ci.com/user/trusty-ci-environment

README.md

@@ -1,5 +1,5 @@
 # pdml2flow [![PyPI version](https://badge.fury.io/py/pdml2flow.svg)](https://badge.fury.io/py/pdml2flow) 
-_Aggregates wireshark pdml to flows_
+_Aggregates wireshark pdml to flows, with plugins_
 
 | Branch  | Build  | Coverage |
 | ------- | ------ | -------- |
@@ -11,24 +11,29 @@ _Aggregates wireshark pdml to flows_
   - 3.4
   - 3.5
   - 3.5-dev
+  - 3.6
+  - 3.6-dev
+  - 3.7-dev
   - nightly
 * [pip](https://pypi.python.org/pypi/pip)
 
 ## Installation
 ```shell
-    $ sudo pip install pdml2flow
+$ sudo pip install pdml2flow
 ```
 
 ## Usage
 ```shell
 $ pdml2flow -h
-usage: pdml2flow [-h] [-f FLOW_DEF_STR] [-t FLOW_BUFFER_TIME] [-l DATA_MAXLEN]
-                 [-s] [-x] [-c] [-a] [-m] [-d] [-p PLUGIN_LOAD] [-0]
+usage: pdml2flow [-h] [--version] [-f FLOW_DEF_STR] [-t FLOW_BUFFER_TIME]
+                 [-l DATA_MAXLEN] [-s] [-c] [-a] [-m] [-d] [+json [args]]
+                 [+xml [args]]
 
 Aggregates wireshark pdml to flows
 
 optional arguments:
   -h, --help           show this help message and exit
+  --version            Print version and exit
   -f FLOW_DEF_STR      Fields which define the flow, nesting with: '.'
                        [default: ['vlan.id', 'ip.src', 'ip.dst', 'ipv6.src',
                        'ipv6.dst', 'udp.stream', 'tcp.stream']]
@@ -38,82 +43,81 @@ optional arguments:
                        200]
   -s                   Extract show names, every data leaf will now look like
                        { raw : [] , show: [] } [default: False]
-  -x                   Switch to xml output [default: False]
   -c                   Removes duplicate data when merging objects, will not
                        preserve order of leaves [default: False]
   -a                   Instead of merging the frames will append them to an
                        array [default: False]
   -m                   Appends flow metadata [default: False]
   -d                   Debug mode [default: False]
-  -p PLUGIN_LOAD       Plguins to load, installed: [] [default: []]
-  -0                   Terminates lines with null character [default: False]
-```
 
-## Example
-Sniff from interface:
-```shell
-$ tshark -i interface -Tpdml | pdml2flow
+Plugins:
+  +json [args]         usage: JSON output [-h] [-0] optional arguments: -h,
+                       --help show this help message and exit -0 Terminates
+                       lines with null character
+  +xml [args]          usage: XML output [-h] [-0] optional arguments: -h,
+                       --help show this help message and exit -0 Terminates
+                       lines with null character
 ```
 
-Write xml output
+## Example
+Sniff from interface and write json:
 ```shell
-$ tshark -i interface -Tpdml | pdml2flow -x
+$ tshark -i interface -Tpdml | pdml2flow +json
 ```
 
 Read a .pcap file
 ```shell
-$ tshark -r pcap_file -Tpdml | pdml2flow
+$ tshark -r pcap_file -Tpdml | pdml2flow +json
 ```
 
 Aggregate based on ethernet source and ethernet destination address
 ```shell
-$ tshark -i interface -Tpdml | pdml2flow -f eth.src -f eth.dst
+$ tshark -i interface -Tpdml | pdml2flow -f eth.src -f eth.dst +json
 ```
 
 Pretty print flows using [jq]
 ```shell
-$ tshark -i interface -Tpdml | pdml2flow | jq
+$ tshark -i interface -Tpdml | pdml2flow +json | jq
 ```
 
 Post-process flows using [FluentFlow]
 ```shell
-$ tshark -i interface -Tpdml | pdml2flow | fluentflow rules.js
+$ tshark -i interface -Tpdml | pdml2flow +json | fluentflow rules.js
 ```
 
-## Utils
-
-The following utils are part of this project
+## Plugins
 
-### pdml2json
-_Converts pdml to json_
+### Create a New Plugin
 
-```shell
-$ pdml2json -h
-usage: pdml2json [-h] [-s] [-d]
+[![asciicast](https://asciinema.org/a/208963.png)](https://asciinema.org/a/208963)
 
-Converts wireshark pdml to json
+## Utils
 
-optional arguments:
-  -h, --help  show this help message and exit
-  -s          Extract show names, every data leaf will now look like { raw :
-              [] , show: [] } [default: False]
-  -d          Debug mode [default: False]
-```
+The following utils are part of this project
 
-### pdml2xml
-_Converts pdml to xml_
+### pdml2frame
+_Wireshark pdml to frames, with plugins_
 
 ```shell
-$ pdml2xml -h
-usage: pdml2xml [-h] [-s] [-d]
+$ pdml2frame -h
+usage: pdml2frame [-h] [--version] [-s] [-d] [+json [args]] [+xml [args]]
 
-Converts wireshark pdml to xml
+Converts wireshark pdml to frames
 
 optional arguments:
-  -h, --help  show this help message and exit
-  -s          Extract show names, every data leaf will now look like { raw :
-              [] , show: [] } [default: False]
-  -d          Debug mode [default: False]
+  -h, --help    show this help message and exit
+  --version     Print version and exit
+  -s            Extract show names, every data leaf will now look like { raw :
+                [] , show: [] } [default: False]
+  -d            Debug mode [default: False]
+
+Plugins:
+  +json [args]  usage: JSON output [-h] [-0] optional arguments: -h, --help
+                show this help message and exit -0 Terminates lines with null
+                character
+  +xml [args]   usage: XML output [-h] [-0] optional arguments: -h, --help
+                show this help message and exit -0 Terminates lines with null
+                character
 ```
 
 [python]: https://www.python.org/

pdml2flow/autovivification.py

@@ -1,15 +1,15 @@
 #!/usr/bin/env python3
 # vim: set fenc=utf8 ts=4 sw=4 et :
-import functools
+from functools import reduce
 
 DEFAULT = object();
 
 class AutoVivification(dict):
 
     def clean_empty(self, d=DEFAULT):
-        """
-        Returns a copy of d without empty leaves
-        see: https://stackoverflow.com/questions/27973988/python-how-to-remove-all-empty-fields-in-a-nested-dict/35263074
+        """Returns a copy of d without empty leaves.
+
+        https://stackoverflow.com/questions/27973988/python-how-to-remove-all-empty-fields-in-a-nested-dict/35263074
         """
         if d is DEFAULT:
             d = self
@@ -55,9 +55,9 @@ def cast_dicts(self, to=DEFAULT, d=DEFAULT):
         return d
 
     def merge(self, b, a=DEFAULT):
-        """
-        merges b into a recursively, if a is not given: merges into self
-        also merges lists and :
+        """Merges b into a recursively, if a is not given: merges into self.
+
+        also merges lists and:
             * merge({a:a},{a:b}) = {a:[a,b]}
             * merge({a:[a]},{a:b}) = {a:[a,b]}
             * merge({a:a},{a:[b]}) = {a:[a,b]}
@@ -83,13 +83,13 @@ def merge(self, b, a=DEFAULT):
         return a
 
     def __getitem__(self, item):
-        """
-        Implementation of perl's autovivification feature.
-        see: https://stackoverflow.com/questions/635483/what-is-the-best-way-to-implement-nested-dictionaries-in-python
+        """Implementation of perl's autovivification feature.
+
+        https://stackoverflow.com/questions/635483/what-is-the-best-way-to-implement-nested-dictionaries-in-python
         """
         # if the item is a list we autoexpand it
         if type(item) is list:
-            return functools.reduce(lambda d, k: d[k], item, self)
+            return reduce(lambda d, k: d[k], item, self)
         else:
             try:
                 return dict.__getitem__(self, item)