fixing/improving minimum install instructions; also changing https->h…

…ttp as it may allow us to avoid ssl dependency

docs/installation.rst

@@ -1,7 +1,6 @@
 Installing Workbench
 ====================
 
-
 Workbench Client
 ----------------
 
@@ -20,10 +19,28 @@ Workbench Server (Minimum Install)
 The workbench server is extremely robust to worker failure. In fact it can run without many of the dependencies
 so you can setup a server quickly with 'Minimum Install' and than later do a 'Full Install'.
 
+Mac/OSX
+~~~~~~~
+
 ::
 
-    $ brew/apt-get install mongodb
-    $ pip install workbench
+    $ brew install mongodb
+
+Ubuntu (14.04 and 12.04)
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    $ sudo apt-get install mongodb
+    $ sudo apt-get install python-dev
+    $ sudo apt-get install g++
+
+Workbench
+~~~~~~~~~
+
+::
+
+    $ pip install workbench --pre
     $ workbench
 
 That's it, the workbench server will come up and is ready to start servicing requests.
@@ -32,10 +49,6 @@ Note: Some workers will fail to load but that is fine, to have all workers run s
 Workbench Server (Full Install)
 -------------------------------
 
-The indexers 'Neo4j' and 'ElasticSearch' are optional. We strongly
-suggest you install both of them but we also appreciate that there are
-cases where that's not possible or feasible.
-
 Mac/OSX
 ~~~~~~~
 
@@ -61,14 +74,17 @@ Ubuntu (14.04 and 12.04)
     - Bro IDS: In general the Bro debian package files are WAY too locked down with dependencies on exact versions of libc6 and python2.6. We have a more 'flexible' version `Bro-2.2-Linux-x86\_64\_flex.deb <https://s3-us-west-2.amazonaws.com/workbench-data/packages/Bro-2.2-Linux-x86_64_flex.deb>`_.
         - sudo dpkg -i Bro-2.2-Linux-x86\_64\_flex.deb
         - If using the Debian package above doesn't work out: 
-            - Check out the Installation tutorial `here <https://www.digitalocean.com/community/tutorials/how-to-install-bro-ids-2-2-on-ubuntu-12-04>`_
-            - or this one `here <http://www.justbeck.com/getting-started-with-bro-ids/>`_ 
+            - Check out the Installation tutorial `bro_install <https://www.digitalocean.com/community/tutorials/how-to-install-bro-ids-2-2-on-ubuntu-12-04>`_
+            - or this one `bro_starting <http://www.justbeck.com/getting-started-with-bro-ids/>`_ 
             - or go to offical Bro Downloads `www.bro.org/download/ <http://www.bro.org/download>`_
 
     .. important:: Put the bro executable in your PATH (/opt/bro/bin or wherever bro is)
 
 Install Indexers
 ~~~~~~~~~~~~~~~~
+The indexers 'Neo4j' and 'ElasticSearch' are optional. We strongly
+suggest you install both of them but we also appreciate that there are
+cases where that's not possible or feasible.
 
 Mac/OSX
 ^^^^^^^
@@ -79,8 +95,7 @@ Mac/OSX
         $ pip install -U elasticsearch
         $ brew install neo4j
     
-        -  Note: You may need to install Java JDK 1.7 `Oracle JDK 1.7 DMG <http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-macosx-x64.dmg>`_
-        for macs.
+    -  Note: You may need to install Java JDK 1.7 `Oracle JDK 1.7 DMG <http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-macosx-x64.dmg>`_ for macs.
 
 Ubuntu (14.04 and 12.04)
 ^^^^^^^^^^^^^^^^^^^^^^^^
@@ -113,7 +128,7 @@ Install Python Modules
 
     ::
     
-        $ pip install workbench
+        $ pip install workbench --pre
         $ workbench
     
     **OR**
@@ -134,8 +149,7 @@ Optional Tools
 
     -  http://robomongo.org/
     -  download and follow install instructions
-    -  create a new connection to localhost (default settings fine). Name it
-    as you wish.
+    -  create a new connection to localhost (default settings fine). Name it as you wish.
 
 Dependency Installation Errors
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

setup.py

@@ -20,12 +20,12 @@
 
 setup(
     name='workbench',
-    version='0.2.5',
+    version='0.2.8',
     description='A medium-data framework for security research and development teams.',
     long_description=readme + '\n\n' + doclink + '\n\n' + history,
-    author='Brian Wylie',
-    author_email='briford@supercowpowers.com',
-    url='https://github.com/SuperCowPowers/workbench',
+    author='The Workbench Team',
+    author_email='support@supercowpowers.com',
+    url='http://github.com/SuperCowPowers/workbench',
     packages=['workbench','workbench.clients', 'workbench.server',
               'workbench.server.bro', 'workbench.workers',
               'workbench.workers.rekall_adapter'],
@@ -36,7 +36,7 @@
     install_requires=['elasticsearch', 'urllib3', 'filemagic', 'pefile', 'py2neo',
                       'pymongo', 'requests', 'cython', 'ssdeep==2.9-0.3', 'watchdog',
                       'yara', 'funcsigs', 'zerorpc', 'distorm3>=0', 'rekall==1.0rc11',
-                      'pytest', 'coverage', 'pytest-cov', 'tox'],
+                      'pytest>=2.5', 'coverage', 'pytest-cov', 'tox'],
     license='MIT',
     zip_safe=False,
     keywords='workbench security python',

workbench/__init__.py

@@ -1,4 +1,4 @@
 ''' Workbench: A medium-data framework for security research and development teams. '''
 __author__ = 'The Workbench Team'
 __email__ = 'support@supercowpowers.com'
-__version__ = '0.2.5'
+__version__ = '0.2.8'

workbench/clients/timeout_corner/stress_test.py

@@ -52,7 +52,7 @@ def process_files(path):
     # Close the workbench connection
     workbench.close()
 
-# Fixme: see https://github.com/SuperCowPowers/workbench/issues/40
+# Fixme: see http://github.com/SuperCowPowers/workbench/issues/40
 def test():
     ''' stress_test test '''
     run()

workbench/server/bro/bro_log_reader.py

@@ -18,18 +18,15 @@ def __init__(self, convert_datetimes=True):
         self.convert_datetimes = convert_datetimes
 
     def read_log(self, logfile):
-        """The read_log method is a generator for rows in a Bro log.
+        """The read_log method returns a memory efficient generator for rows in a Bro log.
 
         Usage: 
             rows = my_bro_reader.read_log(logfile)
             for row in rows:
                 do something with row
-        Because this method returns a generator, it's memory efficient and 
-        does not read the entire file in at once.
 
         Args:
             logfile: The Bro Log file.
-
         """
 
         # Make sure we're at the beginning
@@ -53,7 +50,7 @@ def read_log(self, logfile):
 
     def _parse_bro_header(self, logfile):
         """This method tries to parse the Bro log header section.
-        
+
         Note: My googling is failing me on the documentation on the format,
         so just making a lot of assumptions and skipping some shit.
         Assumption 1: The delimeter is a tab.

workbench/server/workbench.py

@@ -430,7 +430,7 @@ def help(self):
         help_str += '\n\t - Run workbench.help_commands() for command help'
         help_str += '\n\t - Run workbench.help_workers() for a list of workers'
         help_str += '\n\t - Run workbench.help_advanced() for advanced help'
-        help_str += '\n\nSee https://github.com/SuperCowPowers/workbench for more information'
+        help_str += '\n\nSee http://github.com/SuperCowPowers/workbench for more information'
         return help_str
 
     def help_basic(self):
@@ -477,7 +477,7 @@ def help_worker(self, worker):
     def help_advanced(self):
         """ Returns advanced help commands """
         help_str =  '\nWoo! Advanced... <fixme: add documentation for advanced> :)'
-        help_str += '\n\nSee https://github.com/SuperCowPowers/workbench for more information'
+        help_str += '\n\nSee http://github.com/SuperCowPowers/workbench for more information'
         return help_str
 
     def list_all_workers(self):

workbench/workers/mem_base.py

@@ -1,6 +1,6 @@
 
 ''' Memory Image base worker. This worker utilizes the Rekall Memory Forensic Framework.
-    See Google Github: https://github.com/google/rekall
+    See Google Github: http://github.com/google/rekall
     All credit for good stuff goes to them, all credit for bad stuff goes to us. :)
 '''
 
@@ -59,12 +59,12 @@ def test():
     if not os.path.isfile(data_path):
         print 'Not finding exemplar4.mem... Downloading now...'
         import urllib
-        urllib.urlretrieve('https://s3-us-west-2.amazonaws.com/workbench-data/memory_images/exemplar4.vmem', data_path)
+        urllib.urlretrieve('http://s3-us-west-2.amazonaws.com/workbench-data/memory_images/exemplar4.vmem', data_path)
 
     # Did we properly download the memory file?
     if not os.path.isfile(data_path):
         print 'Downloading failed, try it manually...'
-        print 'wget https://s3-us-west-2.amazonaws.com/workbench-data/memory_images/exemplar4.vmem'
+        print 'wget http://s3-us-west-2.amazonaws.com/workbench-data/memory_images/exemplar4.vmem'
         exit(1)
 
     # Store the sample

workbench/workers/mem_connscan.py

@@ -1,6 +1,6 @@
 
 ''' Memory Image ConnScan worker. This worker utilizes the Rekall Memory Forensic Framework.
-    See Google Github: https://github.com/google/rekall
+    See Google Github: http://github.com/google/rekall
     All credit for good stuff goes to them, all credit for bad stuff goes to us. :)
 '''
 import os

workbench/workers/mem_dlllist.py

@@ -1,6 +1,6 @@
 
 ''' Memory Image DllList worker. This worker utilizes the Rekall Memory Forensic Framework.
-    See Google Github: https://github.com/google/rekall
+    See Google Github: http://github.com/google/rekall
     All credit for good stuff goes to them, all credit for bad stuff goes to us. :)
 '''
 import os

workbench/workers/mem_meta.py

@@ -1,6 +1,6 @@
 
 ''' Memory Image Meta worker. This worker utilizes the Rekall Memory Forensic Framework.
-    See Google Github: https://github.com/google/rekall
+    See Google Github: http://github.com/google/rekall
     All credit for good stuff goes to them, all credit for bad stuff goes to us. :)
 '''
 import os

workbench/workers/mem_procdump.py

@@ -1,6 +1,6 @@
 
 ''' Memory Image ProcDump worker. This worker utilizes the Rekall Memory Forensic Framework.
-    See Google Github: https://github.com/google/rekall
+    See Google Github: http://github.com/google/rekall
     All credit for good stuff goes to them, all credit for bad stuff goes to us. :)
 '''
 

workbench/workers/mem_pslist.py

@@ -1,6 +1,6 @@
 
 ''' Memory Image PSList worker. This worker utilizes the Rekall Memory Forensic Framework.
-    See Google Github: https://github.com/google/rekall
+    See Google Github: http://github.com/google/rekall
     All credit for good stuff goes to them, all credit for bad stuff goes to us. :)
 '''
 import os

workbench/workers/pe_indicators.py

@@ -32,9 +32,9 @@
 class PEIndicators(object):
     ''' Create instance of Indicators class. This class uses the
         static features from the pefile module to look for weird stuff.
-        Note: All methods that start with 'check_' will be automatically
-              included as part of the checks that happen when 'execute'
-              is called.
+
+        Note: All methods that start with 'check' will be automatically
+        included as part of the checks that happen when 'execute' is called.
     '''
     dependencies = ['sample']
 
@@ -57,7 +57,7 @@ def execute(self, input_data):
         indicators += [{'description': warn, 'severity': 2, 'category': 'PE_WARN'} 
                        for warn in self.pefile_handle.get_warnings()]
 
-        # Automatically invoke any method of this class that starts with 'check_'
+        # Automatically invoke any method of this class that starts with 'check'
         check_methods = self._get_check_methods()
         for check_method in check_methods:
             hit_data = check_method()
@@ -400,7 +400,7 @@ def _get_check_methods(self):
                 value = getattr(self, key)
             except AttributeError:
                 continue
-            if inspect.ismethod(value) and key.startswith('check_'):
+            if inspect.ismethod(value) and key.startswith('check'):
                 results.append(value)
         return results
 

workbench/workers/rekall_adapter/rekall_adapter.py

@@ -1,6 +1,6 @@
 
 """rekall_adapter: Helps Workbench utilize the Rekall Memory Forensic Framework.
-    See Google Github: https://github.com/google/rekall
+    See Google Github: http://github.com/google/rekall
     All credit for good stuff goes to them, all credit for bad stuff goes to us. :).
 """
 
@@ -214,7 +214,7 @@ def test():
     if not os.path.isfile(data_path):
         print 'Not finding exemplar4.mem... Downloading now...'
         import urllib
-        urllib.urlretrieve('https://s3-us-west-2.amazonaws.com/workbench-data/memory_images/exemplar4.vmem', data_path)
+        urllib.urlretrieve('http://s3-us-west-2.amazonaws.com/workbench-data/memory_images/exemplar4.vmem', data_path)
 
     # Did we properly download the memory file?
     if not os.path.isfile(data_path):

workbench/workers/timeout_corner/swf_meta.py

@@ -1,4 +1,4 @@
-''' SWFMeta worker: This code heavily utilizes https://github.com/timknip/pyswf, all credit for good
+''' SWFMeta worker: This code heavily utilizes http://github.com/timknip/pyswf, all credit for good
                     stuff goes to them, all credit for bad stuff goes to me. :)
 '''
 from swf.movie import SWF

workbench/workers/vt_query.py

@@ -30,7 +30,7 @@ def __init__(self):
     def execute(self, input_data):
         ''' Execute the VTQuery worker '''
         md5 = input_data['meta']['md5']
-        response = requests.get('https://www.virustotal.com/vtapi/v2/file/report', 
+        response = requests.get('http://www.virustotal.com/vtapi/v2/file/report', 
                                 params={'apikey':self.apikey,'resource':md5, 'allinfo':1})
 
         # Make sure we got a json blob back