Merge pull request #43 from kieran-github/development

feat(User-Agent): Fixes #40. Add --user-agent parameter.

.travis.yml

@@ -1,9 +1,9 @@
 language: python
 python:
   - "2.7"
-  - "3.3"
   - "3.4"
   - "3.5"
+
 before_install:
   - sudo apt-get update -qq
   - sudo apt-get install -y -qq python-dev libxslt1-dev libxml2-dev

dscan/common/functions.py

@@ -97,7 +97,7 @@ def dict_combine(x, y):
 def file_len(fname):
     i = 0
     with open(fname) as f:
-        for l in f:
+        for _ in f:
             i += 1
 
     return i

dscan/droopescan_completion

@@ -1,16 +1,16 @@
-_droopescan() 
+_droopescan()
 {
     local cur prev opts commands cms enumerations outputs
     COMPREPLY=()
     cur="${COMP_WORDS[COMP_CWORD]}"
     prev="${COMP_WORDS[COMP_CWORD-1]}"
 
-    opts="--url --url-file --enumerate --help --hide-progressbar --debug-requests --threads --output"
+    opts="--url --url-file --enumerate --help --hide-progressbar --user-agent --debug-requests --threads --output"
     commands="scan stats"
     cms="drupal silverstripe"
     enumerations="p t v u a"
     outputs="json standard"
-    
+
     if [[ ${prev} == "droopescan" || ${prev} == "./droopescan" ]]; then
         COMPREPLY=( $(compgen -W "${commands}" -- ${cur}) )
         return 0

dscan/plugins/internal/base_plugin_internal.py

@@ -95,6 +95,7 @@ def _options(self, pargs):
         output = pargs.output
         timeout = pargs.timeout
         timeout_host = pargs.timeout_host
+        user_agent = pargs.user_agent
         hide_progressbar = pargs.hide_progressbar
         debug_requests = pargs.debug_requests
         follow_redirects = pargs.follow_redirects
@@ -265,7 +266,7 @@ def _general_init(self, opts, out=None):
             self.out.warn(old_req)
 
         self.session.verify = False
-        self.session.headers['User-Agent'] = self.DEFAULT_UA
+        self.session.headers['User-Agent'] = opts["user_agent"]
 
         debug_requests = opts['debug_requests']
         if debug_requests:
@@ -314,7 +315,7 @@ def process_url(self, opts, functionality, enabled_functionality, hide_progressb
 
         output = self.url_scan(url, opts, functionality, enabled_functionality,
                 hide_progressbar=hide_progressbar)
-        
+
         if opts['output'] == "json":
             self._output_json_add_info(output, url)
 
@@ -537,7 +538,7 @@ def _determine_fake_200(self, requests_verb, url):
         response = requests_verb(url + self.not_found_url)
 
         return response.status_code == 200, len(response.content)
-    
+
     def _determine_fake_200_module(self, requests_verb, url_template, url):
         fake_200_url = url_template % (url, self.not_found_module)
         response = requests_verb(fake_200_url)
@@ -686,7 +687,7 @@ def enumerate(self, url, base_url_supplied, scanning_method,
                 except requests.exceptions.ReadTimeout:
                     self.out.warn('\rGot a read timeout. Is the server overloaded? This may affect the results of your scan')
                     continue
-                    
+
                 if r.status_code in expected_status:
                     plugin_url = future_array['plugin_url']
                     plugin_name = future_array['plugin_name']

dscan/plugins/internal/scan.py

@@ -63,6 +63,8 @@ class Meta:
                     the following of redirects.""", dest="follow_redirects", default=True)),
                 (['--host'], dict(action='store', help="""Override host header
                     with this value.""", default=None)),
+                (['--user-agent'], dict(action='store', help='''Override User Agent
+                header performing the scan requests.''', default=BasePluginInternal.DEFAULT_UA, type=str)),
                 (['--massscan-override'], dict(action='store_true',
                     help="""Overrides defaults with defaults convenient for
                     mass-scanning of hosts.""", default=False)),
@@ -128,7 +130,6 @@ def default(self):
 
             opts['url'] = url
             opts['headers'] = self._generate_headers(host_header)
-
             inst.process_url(opts, **inst_dict['kwargs'])
 
         self.out.close()

dscan/tests/__init__.py

@@ -62,7 +62,8 @@ class BaseTest(test.CementTestCase):
         'debug': False,
         'enumerate': 'a',
         'headers': {},
-        'hide_progressbar': False
+        'hide_progressbar': False,
+        'user_agent': 'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0'
     }
 
     host_header = {'Host': 'example.com'}

dscan/tests/base_http_tests.py

@@ -903,7 +903,7 @@ def test_read_timeout_warn(self, warn, _):
 
         result, empty = self.scanner.enumerate_plugins(self.base_url,
                 self.scanner.plugins_base_url, ScanningMethod.forbidden)
-        
+
         assert warn.called
 
     def test_single_site_json_out_cms_fingerprint(self):
@@ -961,3 +961,7 @@ def test_module_fake_200(self, warn, mock):
         assert found_500
         assert not found_200 # 200 should not count as false positive
         assert warn.called
+
+    def test_user_agent_in_session(self):
+        assert 'User-Agent' in self.scanner.session.headers
+        assert self.scanner.session.headers['User-Agent'] == 'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0'

dscan/tests/base_tests.py

@@ -102,7 +102,8 @@ def test_fix_dereference_bug(self):
             'verb': 'a',
             'enumerate': 'p',
             'timeout': 15,
-            'headers': {}
+            'headers': {},
+            'user_agent': 'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0'
         }
         opts_t = dict(opts_p)
         opts_t['enumerate'] = 't'

setup.py

@@ -15,7 +15,7 @@ def read_first_line(f):
 def _post_install():
     os.system("chmod +x /usr/local/bin/droopescan");
 
-class PostInstall(install):  
+class PostInstall(install):
 
     def run(self):
         install.run(self)
@@ -47,8 +47,10 @@ def run(self):
         'cement>=2.6,<2.6.99',
         'requests',
         'pystache',
-        'futures'
     ],
+    extras_require={
+        ':python_version == "2.7"': ['futures']
+    },
     cmdclass={'install': PostInstall}
 )