diff --git a/w3af/core/controllers/core_helpers/strategy.py b/w3af/core/controllers/core_helpers/strategy.py index 83e400f7dd..997ec9e673 100644 --- a/w3af/core/controllers/core_helpers/strategy.py +++ b/w3af/core/controllers/core_helpers/strategy.py @@ -189,7 +189,8 @@ def _fuzzable_request_router(self): self._handle_all_consumer_exceptions(_other) # Route fuzzable requests - route_result = self._route_one_fuzzable_request_batch(_input, output, + route_result = self._route_one_fuzzable_request_batch(_input, + output, finished, consumer_forced_end) diff --git a/w3af/core/controllers/misc/io.py b/w3af/core/controllers/misc/io.py index b59e78688b..62bc444b9d 100644 --- a/w3af/core/controllers/misc/io.py +++ b/w3af/core/controllers/misc/io.py @@ -28,8 +28,8 @@ class NamedStringIO(StringIO, str): def __new__(cls, *args, **kwargs): return super(NamedStringIO, cls).__new__(cls, args[0]) - def __init__(self, thestr, name): - super(NamedStringIO, self).__init__(thestr) + def __init__(self, the_str, name): + super(NamedStringIO, self).__init__(the_str) self._name = name # pylint: disable=E0202 diff --git a/w3af/core/controllers/misc/safe_deepcopy.py b/w3af/core/controllers/misc/safe_deepcopy.py new file mode 100644 index 0000000000..c081b207a6 --- /dev/null +++ b/w3af/core/controllers/misc/safe_deepcopy.py @@ -0,0 +1,44 @@ +""" +safe_deepcopy.py + +Copyright 2015 Andres Riancho + +This file is part of w3af, http://w3af.org/ . + +w3af is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation version 2 of the License. + +w3af is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with w3af; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +""" +import copy + +from w3af.core.controllers.misc.decorators import retry + + +@retry(2, delay=0.5, backoff=1.1) +def safe_deepcopy(instance): + """ + In most cases this will just be a wrapper around copy.deepcopy(instance) + without any added features, but when that fails because of a race condition + such as dictionary changed size during iteration - crawl_plugin.py #8956 , + then we retry. + + I don't want to debug the real issue since it only happen once and I can + live with the retry. + + :see: https://github.com/andresriancho/w3af/issues/8956 + :see: https://github.com/andresriancho/w3af/issues/9449 + + :param instance: The object instance we want to copy + :return: A deep copy of the instance + """ + return copy.deepcopy(instance) \ No newline at end of file diff --git a/w3af/core/controllers/misc/tests/test_io.py b/w3af/core/controllers/misc/tests/test_io.py new file mode 100644 index 0000000000..d3e9bf411d --- /dev/null +++ b/w3af/core/controllers/misc/tests/test_io.py @@ -0,0 +1,32 @@ +""" +test_io.py + +Copyright 2015 Andres Riancho + +This file is part of w3af, http://w3af.org/ . + +w3af is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation version 2 of the License. + +w3af is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with w3af; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +""" +import unittest + +from w3af.core.controllers.misc.io import NamedStringIO + +class TestIO(unittest.TestCase): + + def test_named_string_io(self): + content = 'content' + name = 'name' + ns_io = NamedStringIO(content, name) + self.assertEqual(str(ns_io), content) diff --git a/w3af/core/controllers/plugins/audit_plugin.py b/w3af/core/controllers/plugins/audit_plugin.py index 7f9905e0fa..6b3b6905b6 100644 --- a/w3af/core/controllers/plugins/audit_plugin.py +++ b/w3af/core/controllers/plugins/audit_plugin.py @@ -29,6 +29,7 @@ import w3af.core.controllers.output_manager as om from w3af.core.controllers.plugins.plugin import Plugin +from w3af.core.controllers.misc.safe_deepcopy import safe_deepcopy from w3af.core.data.request.variant_identification import are_variants from w3af.core.controllers.exceptions import FourOhFourDetectionException @@ -133,7 +134,7 @@ def audit_with_copy(self, fuzzable_request, orig_resp): In other words, if one plugins modified the fuzzable request object INSIDE that plugin, I don't want the next plugin to suffer from that. """ - fuzzable_request = copy.deepcopy(fuzzable_request) + fuzzable_request = safe_deepcopy(fuzzable_request) try: with ThreadingTimeout(self.PLUGIN_TIMEOUT, swallow_exc=False): diff --git a/w3af/core/controllers/plugins/crawl_plugin.py b/w3af/core/controllers/plugins/crawl_plugin.py index 4d435e1a7e..0d666aeabf 100644 --- a/w3af/core/controllers/plugins/crawl_plugin.py +++ b/w3af/core/controllers/plugins/crawl_plugin.py @@ -19,13 +19,11 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA """ -import copy - from stopit import ThreadingTimeout, TimeoutException from w3af.core.data.request.fuzzable_request import FuzzableRequest +from w3af.core.controllers.misc.safe_deepcopy import safe_deepcopy from w3af.core.controllers.plugins.plugin import Plugin -from w3af.core.controllers.misc.decorators import retry from w3af.core.controllers.core_helpers.fingerprint_404 import is_404 from w3af.core.controllers.exceptions import (BaseFrameworkException, FourOhFourDetectionException) @@ -114,20 +112,3 @@ def http_get_and_parse(self, url, *args, **kwargs): on_success(http_response, url, *args) return http_response - - -@retry(2, delay=0.5, backoff=1.1) -def safe_deepcopy(instance): - """ - In most cases this will just be a wrapper around copy.deepcopy(instance) - without any added features, but when that fails because of a race condition - such as dictionary changed size during iteration - crawl_plugin.py #8956 , - then we retry. - - I don't want to debug the real issue since it only happen once and I can - live with the retry. - - :param instance: The object instance we want to copy - :return: A deep copy of the instance - """ - return copy.deepcopy(instance) \ No newline at end of file