Adding debugging_id to crawl and infra plugins

w3af/core/controllers/core_helpers/consumers/crawl_infrastructure.py

@@ -27,6 +27,7 @@
 import w3af.core.controllers.output_manager as om
 
 from w3af.core.data.db.variant_db import VariantDB
+from w3af.core.data.fuzzer.utils import rand_alnum
 from w3af.core.data.request.fuzzable_request import FuzzableRequest
 from w3af.core.data.misc.ordered_cached_queue import OrderedCachedQueue
 from w3af.core.data.bloomfilter.scalable_bloom import ScalableBloomFilter
@@ -506,44 +507,50 @@ def _discover_worker(self, function_id, plugin, fuzzable_request):
 
         :return: A list with the newly found fuzzable requests.
         """
-        args = (plugin.get_name(), fuzzable_request.get_uri())
-        om.out.debug('%s.discover(%s)' % args)
+        debugging_id = rand_alnum(8)
+
+        args = (plugin.get_name(), fuzzable_request.get_uri(), debugging_id)
+        om.out.debug('%s.discover(%s, did=%s)' % args)
 
         took_line = TookLine(self._w3af_core,
                              plugin.get_name(),
                              'discover',
-                             debugging_id=None,
+                             debugging_id=debugging_id,
                              method_params={'uri': fuzzable_request.get_uri()})
 
         # Status reporting
         status = self._w3af_core.status
         status.set_running_plugin('crawl', plugin.get_name())
         status.set_current_fuzzable_request('crawl', fuzzable_request)
-        om.out.debug('%s is testing "%s"' % (plugin.get_name(),
-                                             fuzzable_request.get_uri()))
 
         try:
-            result = plugin.discover_wrapper(fuzzable_request)
+            result = plugin.discover_wrapper(fuzzable_request, debugging_id)
         except BaseFrameworkException, e:
-            msg = 'An exception was found while running "%s" with "%s": "%s".'
-            om.out.error(msg % (plugin.get_name(), fuzzable_request), e)
+            msg = 'An exception was found while running "%s" with "%s": "%s" (did: %s)'
+            args = (plugin.get_name(), fuzzable_request, debugging_id)
+            om.out.error(msg % args, e)
         except RunOnce:
             # Some plugins are meant to be run only once
             # that is implemented by raising a RunOnce
             # exception
             self._remove_discovery_plugin(plugin)
         except Exception, e:
-            self.handle_exception(plugin.get_type(), plugin.get_name(),
-                                  fuzzable_request, e)
+            self.handle_exception(plugin.get_type(),
+                                  plugin.get_name(),
+                                  fuzzable_request,
+                                  e)
         else:
             # The plugin output is retrieved and analyzed by the
             # _route_plugin_results method, here we just verify that the plugin
             # result is None (which proves that the plugin respects this part
             # of the API)
             if result is not None:
-                msg = 'The %s plugin did NOT return None.' % plugin.get_name()
-                ve = ValueError(msg)
-                self.handle_exception(plugin.get_type(), plugin.get_name(),
-                                      fuzzable_request, ve)
+                msg = 'The %s plugin did NOT return None (did: %s)'
+                args = (plugin.get_name(), debugging_id)
+                ve = ValueError(msg % args)
+                self.handle_exception(plugin.get_type(),
+                                      plugin.get_name(),
+                                      fuzzable_request,
+                                      ve)
 
         took_line.send()

w3af/core/controllers/core_helpers/not_found/tests/test_get_clean_body.py

@@ -27,7 +27,7 @@
 from w3af.core.data.parsers.doc.url import URL
 from w3af.core.data.url.HTTPResponse import HTTPResponse
 from w3af.core.data.dc.headers import Headers
-from w3af.core.controllers.core_helpers.fingerprint_404 import get_clean_body
+from w3af.core.controllers.core_helpers.not_found.get_clean_body import get_clean_body
 
 
 class TestGetCleanBody(unittest.TestCase):

w3af/core/controllers/plugins/crawl_plugin.py

@@ -37,9 +37,12 @@ class CrawlPlugin(Plugin):
 
     :author: Andres Riancho (andres.riancho@gmail.com)
     """
-    def crawl_wrapper(self, fuzzable_request):
+    def discover_wrapper(self, fuzzable_request, debugging_id):
         """
         Wrapper around the crawl method in order to perform some generic tasks.
+
+        :param fuzzable_request: The target to use for infrastructure plugins.
+        :param debugging_id: A unique identifier for this call to discover()
         """
         om.out.debug('[%s] Crawling "%s"' % (self.get_name(),
                                              fuzzable_request.get_uri()))
@@ -50,7 +53,7 @@ def crawl_wrapper(self, fuzzable_request):
         fuzzable_request_copy = safe_deepcopy(fuzzable_request)
 
         try:
-            return self.crawl(fuzzable_request_copy)
+            return self.crawl(fuzzable_request_copy, debugging_id)
         except FourOhFourDetectionException, ffde:
             # We simply ignore any exceptions we find during the 404 detection
             # process. FYI: This doesn't break the xurllib error handling which
@@ -59,10 +62,11 @@ def crawl_wrapper(self, fuzzable_request):
             # https://github.com/andresriancho/w3af/issues/8949
             om.out.debug('%s' % ffde)
 
-    def crawl(self, fuzzable_request):
+    def crawl(self, fuzzable_request, debugging_id):
         """
         This method MUST be implemented on every plugin.
 
+        :param debugging_id: A unique identifier for this call to discover()
         :param fuzzable_request: Represents an HTTP request, with its URL and
                                  parameters.
 
@@ -72,8 +76,6 @@ def crawl(self, fuzzable_request):
         msg = 'Plugin is not implementing required method crawl'
         raise BaseFrameworkException(msg)
 
-    discover_wrapper = crawl_wrapper
-
     def get_type(self):
         return 'crawl'
 

w3af/core/controllers/plugins/infrastructure_plugin.py

@@ -35,18 +35,21 @@ class InfrastructurePlugin(Plugin):
 
     :author: Andres Riancho (andres.riancho@gmail.com)
     """
-    def discover_wrapper(self, fuzzable_request):
+    def discover_wrapper(self, fuzzable_request, debugging_id):
         """
-        Wrapper around the discover method in order to perform some generic
-        tasks.
+        Wrapper around the discover method to perform generic tasks such
+        as cloning the fuzzable request.
+
+        :param fuzzable_request: The target to use for infrastructure plugins.
+        :param debugging_id: A unique identifier for this call to discover()
         """
         # I copy the fuzzable request, to avoid cross plugin contamination
         # in other words, if one plugin modified the fuzzable request object
         # INSIDE that plugin, I don't want the next plugin to suffer from that
         fuzzable_request_copy = safe_deepcopy(fuzzable_request)
 
         try:
-            return self.discover(fuzzable_request_copy)
+            return self.discover(fuzzable_request_copy, debugging_id)
         except FourOhFourDetectionException, ffde:
             # We simply ignore any exceptions we find during the 404 detection
             # process. FYI: This doesn't break the xurllib error handling which
@@ -55,11 +58,12 @@ def discover_wrapper(self, fuzzable_request):
             # https://github.com/andresriancho/w3af/issues/8949
             om.out.debug('%s' % ffde)
 
-    def discover(self, fuzzable_request):
+    def discover(self, fuzzable_request, debugging_id):
         """
         This method MUST be implemented on every plugin.
 
         :param fuzzable_request: The target to use for infrastructure plugins.
+        :param debugging_id: A unique identifier for this call to discover()
         :return: None. These plugins should store information in the KB. Results
                  from this method will be ignored by the core.
         """

w3af/plugins/crawl/archive_dot_org.py

@@ -56,11 +56,12 @@ def __init__(self):
         # User configured parameters
         self._max_depth = 3
 
-    def crawl(self, fuzzable_request):
+    def crawl(self, fuzzable_request, debugging_id):
         """
         Does a search in archive.org and searches for links on the html. Then
         searches those URLs in the target site. This is a time machine !
 
+        :param debugging_id: A unique identifier for this call to discover()
         :param fuzzable_request: A fuzzable_request instance that contains
                                     (among other things) the URL to test.
         """

w3af/plugins/crawl/bing_spider.py

@@ -42,8 +42,9 @@ def __init__(self):
         self._result_limit = 300
 
     @runonce(exc_class=RunOnce)
-    def crawl(self, fuzzable_request):
+    def crawl(self, fuzzable_request, debugging_id):
         """
+        :param debugging_id: A unique identifier for this call to discover()
         :param fuzzable_request: A fuzzable_request instance that contains
                                  (among other things) the URL to test.
         """

w3af/plugins/crawl/content_negotiation.py

@@ -67,12 +67,13 @@ def __init__(self):
         self._tries_left = 3
         self._content_negotiation_enabled = None
 
-    def crawl(self, fuzzable_request):
+    def crawl(self, fuzzable_request, debugging_id):
         """
         1- Check if HTTP server is vulnerable
         2- Exploit using FuzzableRequest
         3- Perform bruteforce for each new directory
 
+        :param debugging_id: A unique identifier for this call to discover()
         :param fuzzable_request: A fuzzable_request instance that contains
                                 (among other things) the URL to test.
         """

w3af/plugins/crawl/digit_sum.py

@@ -53,11 +53,12 @@ def __init__(self):
         self._fuzz_images = False
         self._max_digit_sections = 4
 
-    def crawl(self, fuzzable_request):
+    def crawl(self, fuzzable_request, debugging_id):
         """
         Searches for new URLs by adding and subtracting numbers to the file
         and the parameters.
 
+        :param debugging_id: A unique identifier for this call to discover()
         :param fuzzable_request: A fuzzable_request instance that contains
                                      (among other things) the URL to test.
         """

w3af/plugins/crawl/dir_file_bruter.py

@@ -64,10 +64,11 @@ def __init__(self):
         self._exec = True
         self._already_tested = DiskSet(table_prefix='dir_file_bruter')
 
-    def crawl(self, fuzzable_request):
+    def crawl(self, fuzzable_request, debugging_id):
         """
         Get the file and parse it.
 
+        :param debugging_id: A unique identifier for this call to discover()
         :param fuzzable_request: A fuzzable_request instance that contains
                                (among other things) the URL to test.
         """

w3af/plugins/crawl/dot_ds_store.py

@@ -53,10 +53,11 @@ def __init__(self):
         # Internal variables
         self._analyzed_dirs = DiskSet()
 
-    def crawl(self, fuzzable_request):
+    def crawl(self, fuzzable_request, debugging_id):
         """
         For every directory, fetch a list of files and analyze the response.
 
+        :param debugging_id: A unique identifier for this call to discover()
         :parameter fuzzable_request: A fuzzable_request instance that contains
                                     (among other things) the URL to test.
         """

w3af/plugins/crawl/dot_listing.py

@@ -47,10 +47,11 @@ def __init__(self):
         # Internal variables
         self._analyzed_dirs = ScalableBloomFilter()
 
-    def crawl(self, fuzzable_request):
+    def crawl(self, fuzzable_request, debugging_id):
         """
         For every directory, fetch the .listing file and analyze the response.
 
+        :param debugging_id: A unique identifier for this call to discover()
         :param fuzzable_request: A fuzzable_request instance that contains
                                     (among other things) the URL to test.
         """

w3af/plugins/crawl/dwsync_xml.py

@@ -46,10 +46,11 @@ def __init__(self):
         # Internal variables
         self._analyzed_dirs = DiskSet()
 
-    def crawl(self, fuzzable_request):
+    def crawl(self, fuzzable_request, debugging_id):
         """
         For every directory, fetch a list of files and analyze the response.
-        
+
+        :param debugging_id: A unique identifier for this call to discover()
         :parameter fuzzable_request: A fuzzable_request instance that contains
                                     (among other things) the URL to test.
         """

w3af/plugins/crawl/find_backdoors.py

@@ -70,11 +70,12 @@ def _read_signatures(self):
 
             yield (line, 'Backdoor signature')
 
-    def crawl(self, fuzzable_request):
+    def crawl(self, fuzzable_request, debugging_id):
         """
         For every directory, fetch a list of shell files and analyze the
         response.
 
+        :param debugging_id: A unique identifier for this call to discover()
         :param fuzzable_request: A fuzzable_request instance that contains
                                     (among other things) the URL to test.
         """

w3af/plugins/crawl/find_captchas.py

@@ -45,10 +45,11 @@ def __init__(self):
 
         self._captchas_found = DiskSet(table_prefix='find_captchas')
 
-    def crawl(self, fuzzable_request):
+    def crawl(self, fuzzable_request, debugging_id):
         """
         Find CAPTCHA images.
 
+        :param debugging_id: A unique identifier for this call to discover()
         :param fuzzable_request: A fuzzable_request instance that contains
                                     (among other things) the URL to test.
         """

w3af/plugins/crawl/find_dvcs.py

@@ -75,10 +75,11 @@ def __init__(self):
                       DVCSTest('CVS/Entries', 'cvs repository', self.cvs_entries),
                       DVCSTest('.cvsignore', 'cvs ignore', self.ignore_file)]
 
-    def crawl(self, fuzzable_request):
+    def crawl(self, fuzzable_request, debugging_id):
         """
         For every directory, fetch a list of files and analyze the response.
 
+        :param debugging_id: A unique identifier for this call to discover()
         :param fuzzable_request: A fuzzable_request instance that contains
                                  (among other things) the URL to test.
         """

w3af/plugins/crawl/genexus_xml.py

@@ -44,10 +44,11 @@ class genexus_xml(CrawlPlugin):
     GENEXUS_DB = ('execute.xml', 'DeveloperMenu.xml')
 
     @runonce(exc_class=RunOnce)
-    def crawl(self, fuzzable_request):
+    def crawl(self, fuzzable_request, debugging_id):
         """
         Get the execute.xml file and parse it.
 
+        :param debugging_id: A unique identifier for this call to discover()
         :param fuzzable_request: A fuzzable_request instance that contains
                                 (among other things) the URL to test.
         """

w3af/plugins/crawl/ghdb.py

@@ -59,8 +59,9 @@ def __init__(self):
         self._result_limit = 300
 
     @runonce(exc_class=RunOnce)
-    def crawl(self, fuzzable_request):
+    def crawl(self, fuzzable_request, debugging_id):
         """
+        :param debugging_id: A unique identifier for this call to discover()
         :param fuzzable_request: A fuzzable_request instance that contains
                                 (among other things) the URL to test.
         """

w3af/plugins/crawl/google_spider.py

@@ -42,8 +42,9 @@ def __init__(self):
         self._result_limit = 300
 
     @runonce(exc_class=RunOnce)
-    def crawl(self, fuzzable_request):
+    def crawl(self, fuzzable_request, debugging_id):
         """
+        :param debugging_id: A unique identifier for this call to discover()
         :param fuzzable_request: A fuzzable_request instance that contains
                                     (among other things) the URL to test.
         """

w3af/plugins/crawl/import_results.py

@@ -49,11 +49,12 @@ def __init__(self):
         self._input_burp = ''
 
     @runonce(exc_class=RunOnce)
-    def crawl(self, fuzzable_request):
+    def crawl(self, fuzzable_request, debugging_id):
         """
         Read the input file, and create the fuzzable_request_list based on that
         information.
 
+        :param debugging_id: A unique identifier for this call to discover()
         :param fuzzable_request: A fuzzable_request instance that contains
                                     (among other things) the URL to test.
                                     In this case it is simply ignored and data