PONY-16 Redo Schema and Fix Tests (#15)

* Create newSchema.yml

Took 18 minutes

* rework schema to YAML and update to new format

* Remove Core reference from pypony.py

Took 25 seconds

* Update requirements.dev.txt

Took 34 seconds

* unused import

Took 3 minutes

* fixing schema rework

* more fixes of schema

* temporary stop until we figure out validation

* Added some TODOs

Figured out how to fix `check_endpoint_coverage()`

Took 15 minutes

* Add some TODOs

* Get the validation working

Took 1 hour 0 minutes

* PONY-19 Fix `context_test.py`

Took 6 minutes

* fix request test

* PONY-25, PONY-22, PONY-27 Fix `step_test.py`, `preprocessing_test.py` and Endpoint Coverage (#11)

* begin fixing localhost step files for preprocessing test

* fix operation coverage

* fix step files and get preprocessing_test.py and step_test.py to work

* remove all references to `endpoint`

* remove more references to `endpoint`

* fix response handling for content types that contain `application/json` (#12)

* PONY-26 fix `validate_test.py` (#13)

* finish fixing the last of the tests (#14)

* cleanup unused imports

* remove local files

Co-authored-by: AJ Rice <53190766+ajrice6713@users.noreply.github.com>

pypony.py

@@ -1,8 +1,6 @@
 import sys
 import traceback
-
 import click
-from actions_toolkit import core
 
 from src.validate import verify_api
 
@@ -30,9 +28,9 @@ def main(spec_file, step_file, fail_fast, verbose):
         verify_api(spec_file, step_file, fail_fast, verbose)
     except BaseException as e:
         if verbose:
-            core.error(traceback.format_exc(), title=e.__class__.__name__)
+            print(traceback.format_exc())
         else:
-            core.error(str(e), title=e.__class__.__name__)
+            print(str(e))
 
         sys.exit(1)
 

requirements.dev.txt

@@ -2,5 +2,4 @@ Django==3.2.9
 django_extensions==3.1.3
 djangorestframework==3.12.4
 Faker==9.8.0
-genbadge[coverage]==1.0.6
 pytest==7.0.1

requirements.txt

@@ -1,6 +1,7 @@
 addict==2.4.0
 click==8.0.3
-jschon==0.7.3
+jsonschema==4.4.0
+jschon==0.8.4
 openapi-spec-validator==0.3.1
 prance==0.21.8.0
 py-openapi-schema-to-json-schema==0.0.3

src/errors.py

@@ -6,22 +6,18 @@
 These errors will nicely format these error messages for logging and stack traces.
 """
 
-# Import errors from models subpackage
-from .models.errors import *
-
-# For type hints
 from .models import Response
 
 
-class UndocumentedEndpointError(BaseException):
+class UndocumentedOperationError(BaseException):
     """
-    Raised when the step file contains endpoints that are not documented in the OpenAPI spec.
+    Raised when the step file contains operations that are not documented in the OpenAPI spec.
     This error will halt further execution of the action.
     """
 
     def __init__(self, undocumented: set[str]):
         super().__init__(
-            f"""The following endpoints are undocumented:
+            f"""The following operations are undocumented:
     {undocumented}"""
         )
 
@@ -36,8 +32,8 @@ def __init__(
         self, achieved_coverage: float, target_coverage: float, uncovered: set[str]
     ):
         super().__init__(
-            f"""The endpoint coverage is {achieved_coverage} but the target is {target_coverage}.
-    The following endpoints are uncovered: {uncovered}"""
+            f"""The operation coverage is {achieved_coverage} but the target is {target_coverage}.
+    The following operations are uncovered: {uncovered}"""
         )
 
 
@@ -53,7 +49,7 @@ def __init__(self, statuses: list, step: Response):
     Expected possible responses:
         {', '.join(list(statuses))}
     Actual response:
-        {step.status_code} {step.reason}: {step.text}"""
+        {step.status_code}"""
         )
 
 

src/models/__init__.py

@@ -8,4 +8,4 @@
 from .response import Response
 from .schema import Schema
 from .singleton import Singleton
-from .step import Step
+from .operation import Operation

src/models/context.py

@@ -2,7 +2,7 @@
 """context.py
 
 Context manages all variables used during validation,
-like system environment variables and responses of previous steps.
+like system environment variables and responses of previous operations.
 """
 
 import os
@@ -18,36 +18,36 @@ class Context(metaclass=Singleton):
     The global context object will be used as a singleton across the entire system.
     """
 
-    _steps = SimpleNamespace()
+    _operations = SimpleNamespace()
 
     @property
-    def steps(self):
-        return self._steps
+    def operations(self):
+        return self._operations
 
-    def add_steps(self, step):
+    def add_operations(self, operation):
         """
-        Adds a Step object as an attribute of `self.steps`
+        Adds a Step object as an attribute of `self.operations`
 
         Args:
-            step (Step): the Step object to add
+            operation (Operation): the Operation object to add
         """
 
-        setattr(self.steps, step.name, step)
+        setattr(self.operations, operation.name, operation)
 
-    def clear_steps(self):
+    def clear_operations(self):
         """
-        Clears all Steps objects from attributes of `self.steps`
+        Clears all Operations objects from attributes of `self.operations`
         """
 
-        self._steps = SimpleNamespace()
+        self._operations = SimpleNamespace()
 
     # noinspection PyMethodMayBeStatic
     def evaluate(self, expression: any) -> any:
         """
         Recursively evaluate nested expressions using depth-first search.
         Eventually the evaluation result as a string is returned.
 
-        The only allowed base contexts are "env" and "steps".
+        The only allowed base contexts are "env" and "operations".
 
         Args:
             expression (str): Object of any type that may contain expression(s)
@@ -87,7 +87,7 @@ def evaluate(self, expression: any) -> any:
                 result = os.environ.get(value.split(".", 1)[1])
                 if result is None:
                     raise EnvironmentVariableError(value)
-            elif base == "steps":
+            elif base == "operations":
                 try:
                     result = eval("self." + value)
                 except AttributeError as e:

src/models/step.py → src/models/operation.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
-"""step.py:
+"""operation.py:
 
-Encapsulates step data from the step file, including name, request, response, and schema for easy access.
+Encapsulates operation data from the step file, including name, request, response, and schema for easy access.
 """
 
 from dataclasses import dataclass
@@ -14,9 +14,9 @@
 
 
 @dataclass
-class Step:
+class Operation:
     """
-    Steps object manages request, response, and schema.
+    Operations object manages request, response, and schema.
     """
 
     name: str
@@ -31,5 +31,4 @@ def verify(self) -> Scope:
         Returns:
             Evaluate the response body and return the complete evaluation result tree.
         """
-
         return self.schema.evaluate(self.response.json())

src/models/request.py

@@ -17,18 +17,25 @@ class Request:
     """
     def __init__(
         self,
+        operation_id: str,
+        status_code: int,
         method: str,
         url: str,
         params: dict = None,
         body: dict = None,
         headers: dict = None,
         auth: dict = None,
+        global_auth: dict = None
+
     ):
+        self.operation_id = operation_id
+        self.status_code = status_code
         self.method = method
         self.url = url
         self.params = params
         self.body = body
         self.headers = headers
+        self.global_auth = global_auth
         self.auth = auth
 
     @property
@@ -61,7 +68,20 @@ def auth(self):
 
     @auth.setter
     def auth(self, value):
-        self._auth = Dict(value if value else {"username": "", "password": ""})
+        if value:
+            self._auth = value
+        elif self.global_auth:
+            self._auth = Dict(self.global_auth)
+        else:
+            self._auth = {"username": "", "password": ""}
+
+    @property
+    def global_auth(self):
+        return self._global_auth
+
+    @global_auth.setter
+    def global_auth(self, value):
+        self._global_auth = value
 
     def evaluate_all(self) -> None:
         """
@@ -76,6 +96,6 @@ def evaluate_all(self) -> None:
         self.url = context.evaluate(self.url)
 
         # Evaluate all Dict object
-        for name in ("params", "body", "headers", "auth"):
+        for name in ("params", "body", "headers", "global_auth",  "auth"):
             attr = self.__getattribute__(name)
             self.__setattr__(name, context.evaluate(attr))

src/models/response.py

@@ -20,7 +20,10 @@ class Response:
 
     def __getattr__(self, item):
         if item == "body":
-            body = self.__response.json()
+            if 'application/json' in self.__response.headers['Content-Type']:
+                body = self.__response.json()
+            else:
+                body = self.__response.content
             return Dict(body) if isinstance(body, dict) else body
 
         return getattr(self.__response, item)

src/models/schema.py

@@ -3,7 +3,6 @@
 
 JSON Schema is a vocabulary that allows you to annotate and validate JSON documents.
 """
-
 from jschon import JSONSchema, JSON
 from jschon.jsonschema import Scope
 
@@ -13,11 +12,21 @@
 class Schema:
     """
     Predefines meta schema and stores JSON schema document model.
+    See https://jschon.readthedocs.io/en/latest/reference/jsonschema.html#jschon.jsonschema.JSONSchema
+    See https://jschon.readthedocs.io/en/latest/examples/recursive_schema_extension.html for example
     """
 
     def __init__(self, schema: dict, meta_schema: str = META_SCHEMA):
-        schema["$schema"] = meta_schema
-        self.schema = JSONSchema(schema)
+        json = {"$schema": meta_schema}
+
+        # TODO: support more content types
+        try:
+            json.update(schema['content']['application/json']['schema'])
+        except KeyError as e:
+            print(e)
+            print('Key not found. Most likely content type of the response is not supported')
+
+        self.schema = JSONSchema(json)
 
     def evaluate(self, json: dict) -> Scope:
         """
@@ -29,5 +38,5 @@ def evaluate(self, json: dict) -> Scope:
         Returns:
             Evaluate a JSON document and return the complete evaluation result tree.
         """
-
+        # inspect(self.schema, methods=True)
         return self.schema.evaluate(JSON(json))

src/models/singleton.py

@@ -4,6 +4,7 @@
 This module is only used by Context to ensure the uniqueness of the global context state.
 """
 
+
 class Singleton(type):
     """
     Restricts the instantiation of a class to one single instance.

src/parsing.py

@@ -6,22 +6,17 @@
 If the input file is not valid, it will raise an exception.
 """
 
-import importlib.resources
-import json
 import os
 
 import yaml
-from jschon import JSON, JSONSchema
-from jschon.jsonschema import OutputFormat
+from jsonschema import validate, ValidationError
 from openapi_spec_validator import validate_spec
 from prance import ResolvingParser, ValidationError
 from prance.util.url import ResolutionError
 from ruamel.yaml.scanner import ScannerError as RuamelScannerError
 from yaml.constructor import SafeConstructor
 from yaml.scanner import ScannerError as PyYAMLScannerError
 
-from .errors import JSONValidatorError
-
 
 def parse_spec(spec_file_path: str) -> dict:
     """Parse OpenAPI spec file to a dictionary.
@@ -76,12 +71,10 @@ def parse_steps(step_file_path: str) -> dict:
     """
 
     try:
-        with importlib.resources.path(
-            __package__, "steps_schema.json"
-        ) as step_schema_file:
-            steps_schema = JSONSchema(json.loads(step_schema_file.read_text()))
+        with open("./src/steps_schema.yml", "r") as step_schema_file:
+            steps_schema = yaml.safe_load(step_schema_file.read())
     except FileNotFoundError as e:
-        raise FileNotFoundError(f"Steps schema file not found") from e
+        raise FileNotFoundError("Steps schema file not found") from e
 
     try:
         with open(step_file_path, "r") as step_file:
@@ -92,11 +85,10 @@ def parse_steps(step_file_path: str) -> dict:
 
             # Load the step file and validate it against the step file schema
             yaml_dict = yaml.safe_load(step_file)
-            result = steps_schema.evaluate(JSON(yaml_dict))
-            result_output = result.output(OutputFormat.BASIC)
+            validate(yaml_dict, steps_schema)
 
-            if not result_output["valid"]:
-                raise JSONValidatorError(result_output["errors"])
+    except ValidationError as e:
+        raise ValidationError(f"Steps file has the following syntax errors: {e} ") from e
     except FileNotFoundError as e:
         raise FileNotFoundError(f"Steps file {step_file_path} not found") from e
     except PyYAMLScannerError as e: