Using regular expressions to extract relative paths from HTTP responses

w3af/plugins/audit/file_upload.py

@@ -36,6 +36,8 @@
 from w3af.core.controllers.misc.io import NamedStringIO
 from w3af.core.controllers.exceptions import BaseFrameworkException
 
+from w3af.core.data.parsers.utils.re_extract import ReExtract
+from w3af.core.data.parsers.doc.url import URL
 from w3af.core.data.constants.file_templates.file_templates import get_template_with_payload
 from w3af.core.data.options.opt_factory import opt_factory
 from w3af.core.data.options.option_list import OptionList
@@ -76,7 +78,7 @@ def __init__(self):
         AuditPlugin.__init__(self)
 
         # Internal attributes
-        self._urls_recently_tested = deque(maxlen=30)
+        self._urls_recently_tested = deque(maxlen=300)
         self._urt_lock = RLock()
 
         # User configured
@@ -145,25 +147,21 @@ def _find_files_by_parsing(self, mutant, mutant_response, debugging_id):
         :param mutant_response: The HTTP response associated with the file upload
         :return: None
         """
-        try:
-            doc_parser = parser_cache.dpc.get_document_parser_for(mutant_response)
-        except BaseFrameworkException:
-            # Failed to find a suitable parser for the document
-            return
+        parser_references = self._get_references_from_parser(mutant_response)
+        re_references = self._get_references_regex(mutant, mutant_response)
 
-        parsed_refs, re_refs = doc_parser.get_references()
-
-        all_references = parsed_refs
-        all_references.extend(re_refs)
+        all_references = set()
+        all_references.update(parser_references)
+        all_references.update(re_references)
 
         to_verify = set()
 
         #
         #   Find the uploaded file in the references!
         #
         for ref in all_references:
+            # This one looks really promising!
             if mutant.uploaded_file_name in ref.url_string:
-                # This one looks really promising!
                 to_verify.add(ref)
 
             # These are just in case...
@@ -209,6 +207,44 @@ def _find_files_by_parsing(self, mutant, mutant_response, debugging_id):
 
         self.worker_pool.map_multi_args(self._confirm_file_upload, args)
 
+    def _get_references_regex(self, mutant, mutant_response):
+        """
+        Apply regular expressions to extract links from the HTTP response body.
+
+        :param mutant: The request used to upload the file
+        :param mutant_response: The HTTP response to parse
+        :return: References (links) found in the HTTP response that end with the
+                 uploaded filename.
+        """
+        # Quick performance improvement
+        if mutant.uploaded_file_name not in mutant_response.get_body():
+            return []
+
+        # Apply the regular expressions and extract links
+        re_extract = ReExtract(mutant_response.get_body(),
+                               mutant_response.get_uri(),
+                               mutant_response.get_charset())
+        re_extract.parse()
+
+        return re_extract.get_references()
+
+    def _get_references_from_parser(self, mutant_response):
+        """
+        :param mutant_response: The HTTP response to parse
+        :return: All references (links) found in the HTTP response
+        """
+        try:
+            doc_parser = parser_cache.dpc.get_document_parser_for(mutant_response)
+        except BaseFrameworkException:
+            # Failed to find a suitable parser for the document
+            return
+
+        parsed_refs, re_refs = doc_parser.get_references()
+
+        all_references = parsed_refs
+        all_references.extend(re_refs)
+        return all_references
+
     def _find_files_by_bruteforce(self, mutant, mutant_response, debugging_id):
         """
         Use the framework's knowledge to find the file in all possible locations
@@ -266,9 +302,12 @@ def _confirm_file_upload(self, path, mutant, http_response, debugging_id):
         desc = 'A file upload to a directory inside the webroot was found at: %s'
         desc %= mutant.found_at()
 
-        v = Vuln.from_mutant('Insecure file upload', desc, severity.HIGH,
+        v = Vuln.from_mutant('Insecure file upload',
+                             desc,
+                             severity.HIGH,
                              [http_response.id, response.id],
-                             self.get_name(), mutant)
+                             self.get_name(),
+                             mutant)
 
         v['file_dest'] = response.get_url()
         v['file_vars'] = mutant.get_file_vars()

w3af/plugins/tests/audit/test_file_upload.py

@@ -152,7 +152,7 @@ def test_parse_response(self):
         cfg = self._run_configs['cfg']
 
         with patch('w3af.core.data.fuzzer.utils.rand_alnum') as rand_alnum_mock:
-            rand_alnum_mock.side_effect = 'B' * 239
+            rand_alnum_mock.return_value = 'B' * 239
 
             self._scan(cfg['target'], cfg['plugins'])
 
@@ -176,9 +176,12 @@ class TestRegexOutputFromUpload(TestParseOutputFromUpload):
           </form>
           """
 
-    RESULT = "Thanks for uploading your file to <pre>../../hackable/uploads/foo.png</pre>"
+    RESULT = "Thanks for uploading your file to <pre>../../hackable/uploads/mockname.png</pre>"
 
-    image_content = 'PNG' + 'B' * 239
+    FILE_CONTENT_RAND = 'w3af.core.data.fuzzer.utils.rand_alnum'
+    IMAGE_CONTENT = 'PNG' + 'B' * 239
+
+    FILENAME_RAND_ALPHA = 'w3af.core.data.constants.file_templates.file_templates.rand_alpha'
 
     MOCK_RESPONSES = [
               MockResponse(url=target_url,
@@ -191,8 +194,25 @@ class TestRegexOutputFromUpload(TestParseOutputFromUpload):
                            content_type='text/html',
                            method='POST', status=200),
 
-              MockResponse(url=target_url + 'hackable/uploads/foo.png',
-                           body=image_content,
+              MockResponse(url=target_url + 'hackable/uploads/mockname.png',
+                           body=IMAGE_CONTENT,
                            content_type='image/png',
                            method='GET', status=200),
     ]
+
+    def test_parse_response(self):
+        with patch(self.FILENAME_RAND_ALPHA) as rand_alpha_mock:
+            rand_alpha_mock.return_value = 'mockname'
+
+            with patch(self.FILE_CONTENT_RAND) as rand_alnum_mock:
+                rand_alnum_mock.return_value = 'B' * 239
+
+                cfg = self._run_configs['cfg']
+                self._scan(cfg['target'], cfg['plugins'])
+
+        fu_vulns = self.kb.get('file_upload', 'file_upload')
+        self.assertEquals(1, len(fu_vulns))
+
+        v = fu_vulns[0]
+        self.assertEquals(v.get_name(), 'Insecure file upload')
+        self.assertEquals(str(v.get_url().get_domain_path()), self.target_url)