Merge branch 'feature/improved-pause-on-error-8852' into develop #8852

w3af/core/controllers/core_helpers/fingerprint_404.py

@@ -444,7 +444,7 @@ def is_404(http_response):
 
 def get_clean_body(response):
     """
-    @see: blind_sqli_response_diff.get_clean_body()
+    @see: BlindSqliResponseDiff.get_clean_body()
 
     Definition of clean in this method:
         - input:

w3af/core/controllers/core_helpers/target.py

@@ -48,8 +48,8 @@ def __init__(self):
 
         # Some internal variables
         self._operating_systems = ['unknown', 'unix', 'windows']
-        self._programming_frameworks = ['unknown', 'php', 'asp', 'asp.net', 'java',
-                                        'jsp', 'cfm', 'ruby', 'perl']
+        self._programming_frameworks = ['unknown', 'php', 'asp', 'asp.net',
+                                        'java', 'jsp', 'cfm', 'ruby', 'perl']
 
     def clear(self):
         cf.cf.save('targets', [])
@@ -75,7 +75,7 @@ def get_options(self):
             self._operating_systems) + ')'
         h = 'This setting is here to enhance w3af performance.'
 
-        # This list "hack" has to be done becase the default value is the one
+        # This list "hack" has to be done because the default value is the one
         # in the first position on the list
         tmp_list = self._operating_systems[:]
         tmp_list.remove(cf.cf.get('target_os'))
@@ -121,8 +121,8 @@ def _verify_url(self, target_url, file_target=True):
 
     def set_options(self, options_list):
         """
-        This method sets all the options that are configured using the user interface
-        generated by the framework using the result of get_options().
+        This method sets all the options that are configured using the user
+        interface generated by the framework using the result of get_options().
 
         :param options_list: A dictionary with the options for the plugin.
         :return: No value is returned.
@@ -163,10 +163,10 @@ def set_options(self, options_list):
                   ' And w3af can only scan one target domain at a time.'
             raise BaseFrameworkException(msg % ', '.join(domain_list))
 
-        # Save in the config, the target URLs, this may be usefull for some plugins.
+        # Save in the config, the target URLs, this may be useful for some
+        # plugins
         cf.cf.save('targets', target_urls)
-        cf.cf.save('target_domains', set([u.get_domain()
-                   for u in target_urls]))
+        cf.cf.save('target_domains', list(set([u.get_domain() for u in target_urls])))
         cf.cf.save('baseURLs', [i.base_url() for i in target_urls])
 
         if target_urls:
@@ -183,13 +183,15 @@ def set_options(self, options_list):
         if os.lower() in self._operating_systems:
             cf.cf.save('target_os', os.lower())
         else:
-            raise BaseFrameworkException('Unknown target operating system: ' + os)
+            msg = u'Unknown target operating system: "%s"'
+            raise BaseFrameworkException(msg % os)
 
         pf = options_list['target_framework'].get_value_str()
         if pf.lower() in self._programming_frameworks:
             cf.cf.save('target_framework', pf.lower())
         else:
-            raise BaseFrameworkException('Unknown target programming framework: ' + pf)
+            msg = u'Unknown target programming framework: "%s"'
+            raise BaseFrameworkException(msg % pf)
 
     def get_name(self):
         return 'target_settings'

w3af/core/controllers/daemons/tests/test_webserver.py

@@ -40,7 +40,7 @@ def setUp(self):
 
         for port in xrange(self.PORT, self.PORT + 15):
             try:
-                start_webserver(self.IP, port, self.tempdir)
+                self.server = start_webserver(self.IP, port, self.tempdir)
             except:
                 pass
             else:
@@ -58,9 +58,14 @@ def _create_file(self):
         test_fh.write(self.TESTSTRING)
         test_fh.close()
 
+    def test_is_down(self):
+        # pylint: disable=E1103
+        self.assertFalse(self.server.is_down())
+        # pylint: enable=E1103
+
     def test_GET_exists(self):
         self._create_file()
-        
+
         url = 'http://%s:%s/foofile.txt' % (self.IP, self.PORT)
         response_body = urllib2.urlopen(url).read()
 

w3af/core/controllers/daemons/webserver.py

@@ -32,9 +32,6 @@
 # Created servers
 _servers = {}
 
-# Server timeout in seconds
-SERVER_TIMEOUT = 3 * 60
-
 
 def is_running(ip, port):
     """
@@ -67,7 +64,7 @@ def __init__(self, server_address, webroot, RequestHandlerClass):
         self.__shutdown_request = False
 
     def is_down(self):
-        return self.__shutdown_request
+        return self.__is_shut_down.is_set()
 
     def serve_forever(self, poll_interval=0.5):
         """Handle one request at a time until shutdown.
@@ -79,23 +76,25 @@ def serve_forever(self, poll_interval=0.5):
         self.__is_shut_down.clear()
         try:
             while not self.__shutdown_request:
-                self.handle_request()
+                self.handle_request(poll_interval=poll_interval)
         finally:
-            ##self.__shutdown_request = False
+            self.__shutdown_request = False
             self.__is_shut_down.set()
 
-    def handle_request(self):
+    def handle_request(self, poll_interval=0.5):
         """Handle one request, possibly blocking."""
 
-        fd_sets = select.select([self], [], [], SERVER_TIMEOUT)
+        fd_sets = select.select([self], [], [], poll_interval)
         if not fd_sets[0]:
             self.server_close()
             self.__shutdown_request = True
             return
+
         try:
             request, client_address = self.get_request()
         except socket.error:
             return
+
         if self.verify_request(request, client_address):
             try:
                 self.process_request(request, client_address)
@@ -104,9 +103,6 @@ def handle_request(self):
                 self.close_request(request)
 
     def server_bind(self):
-        msg = 'Changing socket options of w3afHTTPServer to (socket.SOL_SOCKET'\
-              ', socket.SO_REUSEADDR, 1)'
-        om.out.debug(msg)
         self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
         BaseHTTPServer.HTTPServer.server_bind(self)
 
@@ -138,8 +134,9 @@ def do_GET(self):
             else:
                 try:
                     self.send_response(200)
-                    # This isn't nice, but this is NOT a complete web server implementation
-                    # it is only here to serve some files to "victim" web servers
+                    # This isn't nice, but this is NOT a complete web server
+                    # implementation it is only here to serve some files to
+                    # "victim" web servers
                     content_type, encoding = mimetypes.guess_type(self.path)
                     if content_type is not None:
                         self.send_header('Content-type', content_type)
@@ -168,19 +165,20 @@ def log_message(self, fmt, *args):
 
 
 def start_webserver(ip, port, webroot, handler=w3afWebHandler):
-    """Create a http server deamon. The returned instance is unique for <ip>
+    """Create a http server daemon. The returned instance is unique for <ip>
     and <port>.
 
     :param ip: IP address where to bind
     :param port: Port number
-    :param webroot: webserver's root directory
-    :return: A local webserver instance bound to the requested address (<ip>, <port>)
+    :param webroot: webs erver's root directory
+    :return: A local web server instance bound to the requested address (<ip>, <port>)
     """
     server_thread = _get_inst(ip, port)
 
     if server_thread is None or server_thread.is_down():
         web_server = w3afHTTPServer((ip, port), webroot, handler)
         _servers[(ip, port)] = web_server
+
         # Start server!
         server_thread = threading.Thread(target=web_server.serve_forever)
         server_thread.name = 'WebServer'
@@ -191,10 +189,10 @@ def start_webserver(ip, port, webroot, handler=w3afWebHandler):
 
 
 def start_webserver_any_free_port(ip, webroot, handler=w3afWebHandler):
-    """Create a http server deamon in any free port available.
+    """Create a http server daemon in any free port available.
 
     :param ip: IP address where to bind
-    :param webroot: webserver's root directory
+    :param webroot: web server's root directory
     :return: A local webserver instance and the port where it's listening
     """
     web_server = w3afHTTPServer((ip, 0), webroot, handler)

w3af/core/controllers/delay_detection/aprox_delay.py

@@ -53,3 +53,8 @@ def get_string_for_multiplier(self, multiplier):
     def set_base_multiplier(self, multiplier):
         self._base_multiplier = multiplier
 
+    def __repr__(self):
+        return u'<AproxDelay (fmt:%s, char:%s, reps:%s)>' % (self._delay_fmt,
+                                                             self._delay_char,
+                                                             self._char_reps)
+

w3af/core/controllers/delay_detection/exact_delay.py

@@ -26,8 +26,8 @@ class ExactDelay(object):
     """
     def __init__(self, delay_fmt, delta=0, mult=1):
         """
-        :param delay_fmt: The format that should be use to generate the delay string.
-                          Example: "sleep(%s)".
+        :param delay_fmt: The format that should be use to generate the delay
+                          string. Example: "sleep(%s)".
         """
         self._delay_fmt = delay_fmt
         self._delay_delta = delta
@@ -63,7 +63,7 @@ def set_delay_delta(self, delta):
 
     def set_multiplier(self, mult):
         """
-        Some delays are expressed in miliseconds, so we need to take that into
+        Some delays are expressed in milliseconds, so we need to take that into
         account and let the user define a specific delay with 1000 as multiplier
 
         >>> d = ExactDelay('sleep(%s)', mult=1000)
@@ -73,3 +73,8 @@ def set_multiplier(self, mult):
 
         """
         self._delay_multiplier = mult
+
+    def __repr__(self):
+        return u'<ExactDelay (fmt:%s, delta:%s, mult:%s)>' % (self._delay_fmt,
+                                                              self._delay_delta,
+                                                              self._delay_multiplier)

w3af/core/controllers/delay_detection/exact_delay_controller.py

@@ -99,15 +99,15 @@ def delay_is_controlled(self):
         return True, responses
 
     def _log_success(self, delay, response):
-        msg = '(Test id: %s) Successfully controlled HTTP response delay for'\
-              ' URL %s - parameter "%s" for %s seconds using %r, response'\
-              ' wait time was: %s seconds.'
+        msg = u'(Test id: %s) Successfully controlled HTTP response delay for'\
+              u' URL %s - parameter "%s" for %s seconds using %r, response'\
+              u' wait time was: %s seconds.'
         self._log_generic(msg, delay, response)
 
     def _log_failure(self, delay, response):
-        msg = '(Test id: %s) Failed to control HTTP response delay for'\
-              ' URL %s - parameter "%s" for %s seconds using %r, response'\
-              ' wait time was: %s seconds.'
+        msg = u'(Test id: %s) Failed to control HTTP response delay for'\
+              u' URL %s - parameter "%s" for %s seconds using %r, response'\
+              u' wait time was: %s seconds.'
         self._log_generic(msg, delay, response)
 
     def _log_generic(self, msg, delay, response):
@@ -151,7 +151,7 @@ def delay_for(self, delay, original_wait_time):
         # Test if the delay worked
         current_response_wait_time = response.get_wait_time()
         args = (id(self), upper_bound, current_response_wait_time, lower_bound)
-        out.debug('(Test id: %s) %s > %s > %s' % args)
+        out.debug(u'(Test id: %s) %s > %s > %s' % args)
 
         if upper_bound > current_response_wait_time > lower_bound:
             return True, response

w3af/core/controllers/output_manager/__init__.py

@@ -1,7 +1,27 @@
+"""
+Copyright 2006 Andres Riancho
+
+This file is part of w3af, http://w3af.org/ .
+
+w3af is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation version 2 of the License.
+
+w3af is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with w3af; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+"""
 import logging
 
 from .manager import OutputManager
 from .log_sink import LogSink
+
 from w3af.core.controllers.core_helpers.consumers.constants import POISON_PILL
 
 # https://pypi.python.org/pypi/stopit#logging
@@ -45,9 +65,9 @@ def log_sink_factory(om_queue):
     return out
 
 
-# Create the default manager and out instances, we'll be creating others later
+# Create the default manager and out instances, we'll be creating others later:
 # most likely for the log sink, which will be replaced in each sub-process
 manager = OutputManager()
-manager.start()
 
+# Logs to into the logging process through out.debug() , out.error() , etc.
 out = log_sink_factory(manager.get_in_queue())

w3af/core/controllers/output_manager/manager.py

@@ -24,6 +24,7 @@
 import threading
 
 from multiprocessing.dummy import Process
+from functools import wraps
 
 from w3af import ROOT_PATH
 from w3af.core.controllers.misc.factory import factory
@@ -42,20 +43,25 @@ def start_thread_on_demand(func):
     printed using the om (see functions below), which ends up with unordered
     messages printed to the console.
     """
+    @wraps(func)
     def od_wrapper(*args, **kwds):
         from w3af.core.controllers.output_manager import manager
 
+        if manager.is_alive():
+            return func(*args, **kwds)
+
         with OutputManager.start_lock:
-            if not manager.is_alive():
-                try:
-                    manager.start()
-                except RuntimeError:
-                    # is_alive() doesn't always return the true state of the
-                    # thread, thus we need to add this try/except, see:
-                    #
-                    # https://github.com/andresriancho/w3af/issues/7453
-                    pass
+            try:
+                manager.start()
+            except RuntimeError:
+                # is_alive() doesn't always return the true state of the
+                # thread, thus we need to add this try/except, see:
+                #
+                # https://github.com/andresriancho/w3af/issues/7453
+                pass
+
         return func(*args, **kwds)
+
     return od_wrapper
 
 
@@ -123,12 +129,12 @@ def run(self):
                 break
 
             else:
-                args, kwds = work_unit
+                args, kwargs = work_unit
                 #
                 #    Please note that error handling is done inside:
                 #        _call_output_plugins_action
                 #
-                apply(self._call_output_plugins_action, args, kwds)
+                self._call_output_plugins_action(*args, **kwargs)
 
                 self.in_queue.task_done()
 

w3af/core/controllers/plugins/plugin.py

@@ -244,14 +244,20 @@ class UrlOpenerProxy(object):
     # do it this way since the extended_urllib.py also implements __getattr__
     # to provide PUT, PATCH, etc. methods.
     #
-    # These methods won't be wrapped, mostly because they:
+    # These methods won't be wrapped, mostly because they either:
     #   * Don't return an HTTPResponse
     #   * Don't raise HTTPRequestException
     #
     # I noticed this issue when #8705 was reported
     # https://github.com/andresriancho/w3af/issues/8705
-    NO_WRAPPER_FOR = {'send_clean', 'clear', 'end', 'restart', 'get_headers',
-                      'get_cookies', 'get_remote_file_size', 'add_headers',
+    NO_WRAPPER_FOR = {'send_clean',
+                      'clear',
+                      'end',
+                      'restart',
+                      'get_headers',
+                      'get_cookies',
+                      'get_remote_file_size',
+                      'add_headers',
                       '_check_uri',
                       '_handle_send_socket_error',
                       '_handle_send_urllib_error',