Merge pull request #47 from KissPeter/json_ref_fix

Json ref fix

.github/ISSUE_TEMPLATE/bug_report.md

@@ -8,18 +8,23 @@ assignees: ''
 ---
 
 **Describe the bug**
+
 A clear and concise description of what the bug is.
 
 **APIFuzzer debug log**
+
 Please provide the related APIFuzzer debug log
 
 **Related API definition**
-Please provide the whole (preferred) or the API definition section which causes the issue
+
+Please provide only the minimal but valid API definition which causes the issue
 
 **Software environment (please complete the following information):**
- - OS:
--  Python version:
- - APIFuzzer Version:
+
+- OS:
+- Python version:
+- APIFuzzer Version:
 
 **Additional context**
+
 Add any other context about the problem here.

apifuzzer/__init__.py

@@ -1 +1 @@
-__version__ = '0.9.11'
+__version__ = '0.9.12'

apifuzzer/fuzzer_target/request_base_functions.py

@@ -208,7 +208,7 @@ def expand_path_variables(self, url, path_parameters):
                 self.logger.warning('{} was not in the url: {}, adding it'.format(url_path_parameter, url))
                 tmp_url += '&{}={}'.format(path_parameter, path_value)
             formatted_url = tmp_url
-        self.logger.info('Compiled url in {}, out: {}'.format(url, formatted_url))
+        self.logger.debug('Compiled url in {}, out: {}'.format(url, formatted_url))
         return formatted_url.replace("{", "").replace("}", "").replace("+", "/")
 
     @staticmethod

apifuzzer/openapi_template_generator.py

@@ -1,10 +1,11 @@
 import json
 from urllib.parse import urlparse
 
+from json_ref_dict import materialize, RefDict
+
 from apifuzzer.base_template import BaseTemplate
 from apifuzzer.fuzz_utils import get_sample_data_by_type, get_fuzz_type_by_param_type
 from apifuzzer.move_json_parts import JsonSectionAbove
-from apifuzzer.resolve_json_reference import ResolveReferences
 from apifuzzer.template_generator_base import TemplateGenerator
 from apifuzzer.utils import transform_data_to_bytes, pretty_print, get_logger
 
@@ -36,14 +37,22 @@ def __init__(self, api_definition_url, api_definition_file):
         self.logger = get_logger(self.__class__.__name__)
         self.api_definition_url = api_definition_url
         self.api_definition_file = api_definition_file
-        self.reference_resolver = ResolveReferences(api_definition_url=api_definition_url,
-                                                    api_definition_path=api_definition_file)
-        tmp_api_resources = self.reference_resolver.resolve()
+        tmp_api_resources = self.resolve_json_references()
+        print(f'1>>>>>>>>>>>>{json.dumps(tmp_api_resources, indent=2, sort_keys=True)}')
         self.json_formatter = JsonSectionAbove(tmp_api_resources)
         self.api_resources = self.json_formatter.resolve()
+        print(f'2>>>>>>>>>>>>{json.dumps(self.api_resources, indent=2, sort_keys=True)}')
         with open(f'resolved.json', 'w') as f:
             json.dump(self.api_resources, f, sort_keys=True, indent=2)
 
+    def resolve_json_references(self):
+        if self.api_definition_url:
+            reference = self.api_definition_url
+        else:
+            reference = self.api_definition_file
+        ref = RefDict(reference)
+        return materialize(ref)
+
     @staticmethod
     def _normalize_url(url_in):
         """
@@ -241,11 +250,14 @@ def _compile_base_url_for_swagger(self, alternate_url):
         return _base_url
 
     def _compile_base_url_for_openapi(self, alternate_url):
-        uri = urlparse(self.api_resources.get('servers')[0].get('url'))
+        if self.api_resources.get('servers'):
+            uri = urlparse(self.api_resources.get('servers', [])[0].get('url'))
+        else:
+            uri = urlparse(alternate_url)
         if alternate_url:
             _base_url = "/".join([alternate_url.strip('/'), uri.path.strip('/')])
         else:
-            _base_url = self.api_resources.get('servers')[0].get('url')
+            _base_url = self.api_resources.get('servers', [])[0].get('url')
         return _base_url
 
     def compile_base_url(self, alternate_url):

json_ref_resolve.py

@@ -0,0 +1,16 @@
+from json_ref_dict import materialize, RefDict
+
+f1 = '/home/kissp/git/APIFuzzer/bug1_resolved.json'
+f2 = '/home/kissp/git/APIFuzzer/resolved_mp.json'
+
+
+# schema = materialize(RefDict("https://json-schema.org/draft-04/schema#/"))
+def resolve(f):
+    return materialize(RefDict(f))
+
+
+schema1 = resolve(f1)
+print(len(schema1))
+schema2 = resolve(f2)
+print(len(schema2))
+assert schema1 == schema2, f's1:{schema1.keys()}\n s2:{schema2.keys()}'

requirements.txt

@@ -3,3 +3,5 @@ pycurl==7.43.0.6
 ruamel.yaml==0.16.12
 junit-xml==1.9
 jsonpath-ng==1.5.2
+PyYAML==6.0
+json-ref-dict>=0.7.1

test/test_openapi.py

@@ -49,6 +49,6 @@ def test_v3_post_with_schema_multiparm_formdata_content_type(self):
             }
         }
         last_call = self.fuzz_openapi_and_get_last_call(api_path, api_def, schema_definitions=self.schema)
-        assert not isinstance(last_call['req_form']['category_id'], int), last_call
+        assert not isinstance(last_call['req_form'].get('category_id'), int), last_call
         assert last_call['req_headers']['Content-Type'].startswith('multipart/form-data')
         self.repot_basic_check()

test/test_swagger.py

@@ -46,7 +46,7 @@ def test_single_query_string(self):
         # last_call test field sample:
         # 'http://127.0.0.1:5000/query?integer_id=%10'
         last_call = self.fuzz_swagger_and_get_last_call(api_path, api_def)
-        _, last_value_sent = last_call['req_url'].split("=")
+        _, last_value_sent = last_call['req_url'].split("=", 1)
         assert not isinstance(last_value_sent, int), last_call['req_url']
         self.repot_basic_check()