Merge pull request #149 from SuperCowPowers/fix-gevent-coverage

Still issues with getting gevent coverage reported correctly, but taking steps in the right direction.

test/conftest.py

@@ -4,12 +4,11 @@
 
 import zerorpc
 import pytest
-import workbench_single
+import workbench_single # This spins up the server
 
 @pytest.fixture(scope='session')
 def workbench_conn(request):
     ''' Workbench Fixture '''
-    import workbench_single # This spins up the server
 
     # Connect to the server
     conn = zerorpc.Client(timeout=300)
@@ -20,14 +19,3 @@ def workbench_conn(request):
 
     # Return the connection
     return conn
-
-def pytest_generate_tests(metafunc):
-    if 'worker' in metafunc.fixturenames:
-        print 'Getting worker list...'
-        # Connect to the server
-        conn = zerorpc.Client()
-        conn.connect('tcp://localhost:4242')
-        workers = conn.list_all_workers()
-        metafunc.parametrize('worker', workers)
-    else:
-        print 'WTF %s' % metafunc.fixturenames

test/test_workbench.py

@@ -3,9 +3,10 @@
 """
 
 class TestWorkbench(object):
-    ''' Test Workbench Class '''
+    ''' Test Workbench Server '''
 
-    def test_worker(self, worker, workbench_conn):
-        ''' Test a specific worker '''
-        print 'Testing %s...' % worker
-        assert workbench_conn.test_worker(worker)
+    def test_server(self, workbench_conn):
+
+        ''' Test the workbench Server '''
+        print '\nStarting up the Workbench server...'
+        return True

tox.ini

@@ -12,8 +12,13 @@ deps =
     pytest-cov
     pyjack
 commands =
-    py.test -v test/test_workbench.py --doctest-module --cov=workbench --cov-report term-missing
+    py.test -v test/test_workbench.py workbench/workers workbench/clients --doctest-module --cov=workbench --cov-report term-missing
 
+[pytest]
+adopts=--doctest-modules
+python_files=*.py
+python_functions=test
+norecursedirs=.tox .git experimental timeout_corner
 
 [testenv:docs]
 changedir=docs/

workbench/clients/customer_report.py

@@ -1,24 +1,18 @@
 ''' This client generates customer reports on all the samples in workbench '''
 import zerorpc
 import pprint
-import argparse
+import os
 import ConfigParser
 
 def main():
     ''' This client generates customer reports on all the samples in workbench '''
 
     # Grab server info from configuration file
     workbench_conf = ConfigParser.ConfigParser()
-    workbench_conf.read('config.ini')
+    config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config.ini')
+    workbench_conf.read(config_path)
     server = workbench_conf.get('workbench', 'server_uri') 
-    port = workbench_conf.getint('workbench', 'server_port') 
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-p', '--port', type=int, default=port, help='port used by workbench server')
-    parser.add_argument('-s', '--server', type=str, default=server, help='location of workbench server')
-    args = parser.parse_args()
-    port = str(args.port)
-    server = str(args.server)
+    port = workbench_conf.get('workbench', 'server_port')
 
     # Start up workbench connection
     workbench = zerorpc.Client()

workbench/clients/log_meta_stream.py

@@ -2,31 +2,25 @@
 import zerorpc
 import os
 import pprint
-import argparse
 import ConfigParser
 
 def main():
     ''' This client get meta data about log files '''
 
     # Grab server info from configuration file
     workbench_conf = ConfigParser.ConfigParser()
-    workbench_conf.read('config.ini')
+    config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config.ini')
+    workbench_conf.read(config_path)
     server = workbench_conf.get('workbench', 'server_uri') 
-    port = workbench_conf.getint('workbench', 'server_port') 
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-p', '--port', type=int, default=port, help='port used by workbench server')
-    parser.add_argument('-s', '--server', type=str, default=server, help='location of workbench server')
-    args = parser.parse_args()
-    port = str(args.port)
-    server = str(args.server)
+    port = workbench_conf.get('workbench', 'server_port')
 
     # Start up workbench connection
     workbench = zerorpc.Client()
     workbench.connect('tcp://'+server+':'+port)
 
     # Test out some log files
-    file_list = [os.path.join('../data/log', child) for child in os.listdir('../data/log')]
+    data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'../data/log')
+    file_list = [os.path.join(data_path, child) for child in os.listdir(data_path)]
     for filename in file_list:
         with open(filename,'rb') as f:
 

workbench/clients/pcap_bro_indexer.py

@@ -1,7 +1,6 @@
 ''' This client pushes PCAPs -> Bro -> ELS Indexer '''
 import zerorpc
 import os
-import argparse
 import ConfigParser
 
 
@@ -10,23 +9,18 @@ def main():
 
     # Grab server info from configuration file
     workbench_conf = ConfigParser.ConfigParser()
-    workbench_conf.read('config.ini')
+    config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config.ini')
+    workbench_conf.read(config_path)
     server = workbench_conf.get('workbench', 'server_uri') 
-    port = workbench_conf.getint('workbench', 'server_port') 
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-p', '--port', type=int, default=port, help='port used by workbench server')
-    parser.add_argument('-s', '--server', type=str, default=server, help='location of workbench server')
-    args = parser.parse_args()
-    port = str(args.port)
-    server = str(args.server)
+    port = workbench_conf.get('workbench', 'server_port')
 
     # Start up workbench connection
     workbench = zerorpc.Client(timeout=300)
     workbench.connect('tcp://'+server+':'+port)
 
     # Test out getting the raw Bro logs from a PCAP file and sending results to an ELS indexer
-    file_list = [os.path.join('../data/pcap', child) for child in os.listdir('../data/pcap')]
+    data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'../data/pcap')
+    file_list = [os.path.join(data_path, child) for child in os.listdir(data_path)]
     for filename in file_list:
 
         # Skip OS generated files

workbench/clients/pcap_bro_raw.py

@@ -1,24 +1,17 @@
 ''' This client gets the raw bro logs from PCAP files '''
 import zerorpc
 import os
-import argparse
 import ConfigParser
 
 def main():
     ''' This client gets the raw bro logs from PCAP files '''
 
     # Grab server info from configuration file
     workbench_conf = ConfigParser.ConfigParser()
-    workbench_conf.read('config.ini')
+    config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config.ini')
+    workbench_conf.read(config_path)
     server = workbench_conf.get('workbench', 'server_uri') 
-    port = workbench_conf.getint('workbench', 'server_port') 
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-p', '--port', type=int, default=port, help='port used by workbench server')
-    parser.add_argument('-s', '--server', type=str, default=server, help='location of workbench server')
-    args = parser.parse_args()
-    port = str(args.port)
-    server = str(args.server)
+    port = workbench_conf.get('workbench', 'server_port')
 
     # Start up workbench connection
     workbench = zerorpc.Client(timeout=300)
@@ -28,7 +21,8 @@ def main():
     # Test out getting the raw Bro logs from a PCAP file
     # Note: you can get a super nice 'generator' python list of dict by using
     #       'stream_sample' instead of 'get_sample'.
-    file_list = [os.path.join('../data/pcap', child) for child in os.listdir('../data/pcap')]
+    data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'../data/pcap')
+    file_list = [os.path.join(data_path, child) for child in os.listdir(data_path)]
     for filename in file_list:
 
         # Skip OS generated files

workbench/clients/pcap_bro_urls.py

@@ -2,32 +2,26 @@
 import zerorpc
 import os
 import pprint
-import argparse
 import ConfigParser
 
 def main():
     ''' This client gets extracts URLs from PCAP files (via Bro logs) '''
 
     # Grab server info from configuration file
     workbench_conf = ConfigParser.ConfigParser()
-    workbench_conf.read('config.ini')
+    config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config.ini')
+    workbench_conf.read(config_path)
     server = workbench_conf.get('workbench', 'server_uri') 
-    port = workbench_conf.getint('workbench', 'server_port') 
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-p', '--port', type=int, default=port, help='port used by workbench server')
-    parser.add_argument('-s', '--server', type=str, default=server, help='location of workbench server')
-    args = parser.parse_args()
-    port = str(args.port)
-    server = str(args.server)
+    port = workbench_conf.get('workbench', 'server_port')
 
     # Start up workbench connection
     workbench = zerorpc.Client(timeout=300)
     workbench.connect('tcp://'+server+':'+port)
 
     # Loop through all the pcaps and collect a set of urls(hosts) from the http_log files
     urls = set()
-    file_list = [os.path.join('../data/pcap', child) for child in os.listdir('../data/pcap')]
+    data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'../data/pcap')
+    file_list = [os.path.join(data_path, child) for child in os.listdir(data_path)]
     for filename in file_list:
 
         # Skip OS generated files

workbench/clients/pcap_bro_view.py

@@ -2,24 +2,17 @@
 import zerorpc
 import os
 import pprint
-import argparse
 import ConfigParser
 
 def main():
     ''' This client pulls PCAP 'views' (view summarize what's in a sample) '''
 
     # Grab server info from configuration file
     workbench_conf = ConfigParser.ConfigParser()
-    workbench_conf.read('config.ini')
+    config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config.ini')
+    workbench_conf.read(config_path)
     server = workbench_conf.get('workbench', 'server_uri') 
-    port = workbench_conf.getint('workbench', 'server_port') 
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-p', '--port', type=int, default=port, help='port used by workbench server')
-    parser.add_argument('-s', '--server', type=str, default=server, help='location of workbench server')
-    args = parser.parse_args()
-    port = str(args.port)
-    server = str(args.server)
+    port = workbench_conf.get('workbench', 'server_port')
 
     # Start up workbench connection
     workbench = zerorpc.Client(timeout=300)
@@ -28,7 +21,8 @@ def main():
     # Test out getting the raw Bro logs from a PCAP file
     # Note: you can get a super nice 'generator' python list of dict by using
     #       'stream_sample' instead of 'get_sample'.
-    file_list = [os.path.join('../data/pcap', child) for child in os.listdir('../data/pcap')]
+    data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'../data/pcap')
+    file_list = [os.path.join(data_path, child) for child in os.listdir(data_path)]
     for filename in file_list:
 
         # Skip OS generated files

workbench/clients/pcap_meta.py

@@ -2,31 +2,25 @@
 import zerorpc
 import os
 import pprint
-import argparse
 import ConfigParser
 
 def main():
     ''' This client pulls PCAP meta data '''
 
     # Grab server info from configuration file
     workbench_conf = ConfigParser.ConfigParser()
-    workbench_conf.read('config.ini')
+    config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config.ini')
+    workbench_conf.read(config_path)
     server = workbench_conf.get('workbench', 'server_uri') 
-    port = workbench_conf.getint('workbench', 'server_port') 
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-p', '--port', type=int, default=port, help='port used by workbench server')
-    parser.add_argument('-s', '--server', type=str, default=server, help='location of workbench server')
-    args = parser.parse_args()
-    port = str(args.port)
-    server = str(args.server)
+    port = workbench_conf.get('workbench', 'server_port')
 
     # Start up workbench connection
     workbench = zerorpc.Client()
     workbench.connect('tcp://'+server+':'+port)
 
     # Test out PCAP data
-    file_list = [os.path.join('../data/pcap', child) for child in os.listdir('../data/pcap')]
+    data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'../data/pcap')
+    file_list = [os.path.join(data_path, child) for child in os.listdir(data_path)]
     for filename in file_list:
 
         # Skip OS generated files

workbench/clients/pcap_meta_indexer.py

@@ -1,31 +1,25 @@
 ''' This client pushes PCAPs -> MetaDaa -> ELS Indexer '''
 import zerorpc
 import os
-import argparse
 import ConfigParser
 
 def main():
     ''' This client pushes PCAPs -> MetaDaa -> ELS Indexer '''
 
     # Grab server info from configuration file
     workbench_conf = ConfigParser.ConfigParser()
-    workbench_conf.read('config.ini')
+    config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config.ini')
+    workbench_conf.read(config_path)
     server = workbench_conf.get('workbench', 'server_uri') 
-    port = workbench_conf.getint('workbench', 'server_port') 
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-p', '--port', type=int, default=port, help='port used by workbench server')
-    parser.add_argument('-s', '--server', type=str, default=server, help='location of workbench server')
-    args = parser.parse_args()
-    port = str(args.port)
-    server = str(args.server)
+    port = workbench_conf.get('workbench', 'server_port')
 
     # Start up workbench connection
     workbench = zerorpc.Client()
     workbench.connect('tcp://'+server+':'+port)
 
     # Test out PCAP data
-    file_list = [os.path.join('../data/pcap', child) for child in os.listdir('../data/pcap')]
+    data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'../data/pcap')
+    file_list = [os.path.join(data_path, child) for child in os.listdir(data_path)]
     for filename in file_list:
 
         # Skip OS generated files

workbench/clients/pe_indexer.py

@@ -1,6 +1,5 @@
 ''' This client pushes PE Files -> ELS Indexer '''
 import zerorpc
-import argparse
 import os
 import pprint
 import ConfigParser
@@ -10,23 +9,18 @@ def main():
 
     # Grab server info from configuration file
     workbench_conf = ConfigParser.ConfigParser()
-    workbench_conf.read('config.ini')
+    config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config.ini')
+    workbench_conf.read(config_path)
     server = workbench_conf.get('workbench', 'server_uri') 
-    port = workbench_conf.getint('workbench', 'server_port') 
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-p', '--port', type=int, default=port, help='port used by workbench server')
-    parser.add_argument('-s', '--server', type=str, default=server, help='location of workbench server')
-    args = parser.parse_args()
-    port = str(args.port)
-    server = str(args.server)
+    port = workbench_conf.get('workbench', 'server_port')
 
     # Start up workbench connection
     workbench = zerorpc.Client()
     workbench.connect('tcp://'+server+':'+port)
 
     # Test out PEFile -> strings -> indexer -> search
-    file_list = [os.path.join('../data/pe/bad', child) for child in os.listdir('../data/pe/bad')]
+    data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'../data/pe/bad')
+    file_list = [os.path.join(data_path, child) for child in os.listdir(data_path)]
     for filename in file_list:
 
         # Skip OS generated files