OpenAPI.can_parse() now returns true even when the content-type is no…

…t in the expected list

w3af/core/data/parsers/doc/open_api/main.py

@@ -62,7 +62,8 @@ class OpenAPI(BaseParser):
                      'text/yaml',
                      'text/x-yaml',
                      'application/yaml',
-                     'application/x-yaml',)
+                     'application/x-yaml',
+                     'application/octet-stream')
 
     KEYWORDS = ('consumes',
                 'produces',
@@ -132,6 +133,14 @@ def is_valid_json_or_yaml(http_resp):
         else:
             return True
 
+    @staticmethod
+    def looks_like_json_or_yaml(http_resp):
+        """
+        :param http_resp: The HTTP response we want to parse
+        :return: True if it seems that this response body holds JSON or YAML
+        """
+        return ':' in '\n'.join(http_resp.body.split('\n')[:20])
+
     @staticmethod
     def can_parse(http_resp):
         """
@@ -140,8 +149,23 @@ def can_parse(http_resp):
 
         :return: True if it seems that the HTTP response contains an Open API spec
         """
-        # Only parse JSON and YAML
-        if not OpenAPI.content_type_match(http_resp):
+        #
+        # In the past we had this check:
+        #
+        # if not OpenAPI.content_type_match(http_resp):
+        #     return False
+        #
+        # But real-life testing showed that it was too restrictive. Some web
+        # servers and frameworks did not return the "expected" content-types
+        # which triggered bugs in can_parse()
+        #
+        # Had to replace it with two other checks, which is worse in performance,
+        # more permissive, but should fix the bug
+        #
+        if http_resp.is_image():
+            return False
+
+        if not OpenAPI.looks_like_json_or_yaml(http_resp):
             return False
 
         # Only parse documents that look like Open API docs

w3af/core/data/parsers/doc/open_api/tests/test_main.py

@@ -60,6 +60,8 @@ def test_json_pet_store(self):
                                 URL('http://moth/swagger.json'),
                                 _id=1)
 
+        self.assertTrue(OpenAPI.can_parse(response))
+
         parser = OpenAPI(response)
         parser.parse()
         api_calls = parser.get_api_calls()
@@ -135,6 +137,8 @@ def test_json_multiple_paths_and_headers(self):
                                 URL('http://moth/swagger.json'),
                                 _id=1)
 
+        self.assertTrue(OpenAPI.can_parse(response))
+
         parser = OpenAPI(response)
         parser.parse()
         api_calls = parser.get_api_calls()
@@ -219,6 +223,8 @@ def test_custom_content_type(self):
                                 URL('http://moth/swagger.json'),
                                 _id=1)
 
+        self.assertTrue(OpenAPI.can_parse(response))
+
         parser = OpenAPI(response)
         parser.parse()
         api_calls = parser.get_api_calls()
@@ -277,6 +283,8 @@ def test_unknown_content_type(self):
                                 URL('http://moth/swagger.json'),
                                 _id=1)
 
+        self.assertTrue(OpenAPI.can_parse(response))
+
         parser = OpenAPI(response)
         parser.parse()
         api_calls = parser.get_api_calls()
@@ -292,6 +300,8 @@ def test_large_many_endpoints(self):
                                 URL('http://moth/swagger.json'),
                                 _id=1)
 
+        self.assertTrue(OpenAPI.can_parse(response))
+
         #
         # In some cases with validation enabled (not the default) we find a set
         # of endpoints:
@@ -328,6 +338,8 @@ def test_disabling_headers_discovery(self):
                                 URL('http://moth/swagger.json'),
                                 _id=1)
 
+        self.assertTrue(OpenAPI.can_parse(response))
+
         parser = OpenAPI(response, discover_fuzzable_headers=False)
         parser.parse()
         api_calls = parser.get_api_calls()
@@ -409,6 +421,8 @@ def test_disabling_spec_validation(self):
                                 URL('http://moth/swagger.json'),
                                 _id=1)
 
+        self.assertTrue(OpenAPI.can_parse(response))
+
         #
         # By default we don't validate the swagger spec, which allows us to
         # parse some invalid specs and extract information
@@ -447,6 +461,8 @@ def test_real_api_yaml(self):
                                 URL('http://moth/swagger.yaml'),
                                 _id=1)
 
+        self.assertTrue(OpenAPI.can_parse(response))
+
         parser = OpenAPI(response)
         parser.parse()
         api_calls = parser.get_api_calls()
@@ -627,6 +643,8 @@ def test_issue_210(self):
                                 URL('http://moth/swagger.yaml'),
                                 _id=1)
 
+        self.assertTrue(OpenAPI.can_parse(response))
+
         parser = OpenAPI(response)
         parser.parse()
         api_calls = parser.get_api_calls()