diff --git a/pypony.py b/pypony.py index 3ae4105..fad9efc 100644 --- a/pypony.py +++ b/pypony.py @@ -1,8 +1,6 @@ import sys import traceback - import click -from actions_toolkit import core from src.validate import verify_api @@ -30,9 +28,9 @@ def main(spec_file, step_file, fail_fast, verbose): verify_api(spec_file, step_file, fail_fast, verbose) except BaseException as e: if verbose: - core.error(traceback.format_exc(), title=e.__class__.__name__) + print(traceback.format_exc()) else: - core.error(str(e), title=e.__class__.__name__) + print(str(e)) sys.exit(1) diff --git a/requirements.dev.txt b/requirements.dev.txt index e282d2d..287298e 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -2,5 +2,4 @@ Django==3.2.9 django_extensions==3.1.3 djangorestframework==3.12.4 Faker==9.8.0 -genbadge[coverage]==1.0.6 pytest==7.0.1 diff --git a/requirements.txt b/requirements.txt index 7ce0a7e..3b4e21f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ addict==2.4.0 click==8.0.3 -jschon==0.7.3 +jsonschema==4.4.0 +jschon==0.8.4 openapi-spec-validator==0.3.1 prance==0.21.8.0 py-openapi-schema-to-json-schema==0.0.3 diff --git a/src/errors.py b/src/errors.py index d240ad1..3998b24 100644 --- a/src/errors.py +++ b/src/errors.py @@ -6,22 +6,18 @@ These errors will nicely format these error messages for logging and stack traces. """ -# Import errors from models subpackage -from .models.errors import * - -# For type hints from .models import Response -class UndocumentedEndpointError(BaseException): +class UndocumentedOperationError(BaseException): """ - Raised when the step file contains endpoints that are not documented in the OpenAPI spec. + Raised when the step file contains operations that are not documented in the OpenAPI spec. This error will halt further execution of the action. """ def __init__(self, undocumented: set[str]): super().__init__( - f"""The following endpoints are undocumented: + f"""The following operations are undocumented: {undocumented}""" ) @@ -36,8 +32,8 @@ def __init__( self, achieved_coverage: float, target_coverage: float, uncovered: set[str] ): super().__init__( - f"""The endpoint coverage is {achieved_coverage} but the target is {target_coverage}. - The following endpoints are uncovered: {uncovered}""" + f"""The operation coverage is {achieved_coverage} but the target is {target_coverage}. + The following operations are uncovered: {uncovered}""" ) @@ -53,7 +49,7 @@ def __init__(self, statuses: list, step: Response): Expected possible responses: {', '.join(list(statuses))} Actual response: - {step.status_code} {step.reason}: {step.text}""" + {step.status_code}""" ) diff --git a/src/models/__init__.py b/src/models/__init__.py index 6673834..c29b9f1 100644 --- a/src/models/__init__.py +++ b/src/models/__init__.py @@ -8,4 +8,4 @@ from .response import Response from .schema import Schema from .singleton import Singleton -from .step import Step +from .operation import Operation diff --git a/src/models/context.py b/src/models/context.py index b777bad..a06a8bd 100644 --- a/src/models/context.py +++ b/src/models/context.py @@ -2,7 +2,7 @@ """context.py Context manages all variables used during validation, -like system environment variables and responses of previous steps. +like system environment variables and responses of previous operations. """ import os @@ -18,28 +18,28 @@ class Context(metaclass=Singleton): The global context object will be used as a singleton across the entire system. """ - _steps = SimpleNamespace() + _operations = SimpleNamespace() @property - def steps(self): - return self._steps + def operations(self): + return self._operations - def add_steps(self, step): + def add_operations(self, operation): """ - Adds a Step object as an attribute of `self.steps` + Adds a Step object as an attribute of `self.operations` Args: - step (Step): the Step object to add + operation (Operation): the Operation object to add """ - setattr(self.steps, step.name, step) + setattr(self.operations, operation.name, operation) - def clear_steps(self): + def clear_operations(self): """ - Clears all Steps objects from attributes of `self.steps` + Clears all Operations objects from attributes of `self.operations` """ - self._steps = SimpleNamespace() + self._operations = SimpleNamespace() # noinspection PyMethodMayBeStatic def evaluate(self, expression: any) -> any: @@ -47,7 +47,7 @@ def evaluate(self, expression: any) -> any: Recursively evaluate nested expressions using depth-first search. Eventually the evaluation result as a string is returned. - The only allowed base contexts are "env" and "steps". + The only allowed base contexts are "env" and "operations". Args: expression (str): Object of any type that may contain expression(s) @@ -87,7 +87,7 @@ def evaluate(self, expression: any) -> any: result = os.environ.get(value.split(".", 1)[1]) if result is None: raise EnvironmentVariableError(value) - elif base == "steps": + elif base == "operations": try: result = eval("self." + value) except AttributeError as e: diff --git a/src/models/step.py b/src/models/operation.py similarity index 73% rename from src/models/step.py rename to src/models/operation.py index 72dffa4..77954de 100644 --- a/src/models/step.py +++ b/src/models/operation.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -"""step.py: +"""operation.py: -Encapsulates step data from the step file, including name, request, response, and schema for easy access. +Encapsulates operation data from the step file, including name, request, response, and schema for easy access. """ from dataclasses import dataclass @@ -14,9 +14,9 @@ @dataclass -class Step: +class Operation: """ - Steps object manages request, response, and schema. + Operations object manages request, response, and schema. """ name: str @@ -31,5 +31,4 @@ def verify(self) -> Scope: Returns: Evaluate the response body and return the complete evaluation result tree. """ - return self.schema.evaluate(self.response.json()) diff --git a/src/models/request.py b/src/models/request.py index 5132d68..5ea9daf 100644 --- a/src/models/request.py +++ b/src/models/request.py @@ -17,18 +17,25 @@ class Request: """ def __init__( self, + operation_id: str, + status_code: int, method: str, url: str, params: dict = None, body: dict = None, headers: dict = None, auth: dict = None, + global_auth: dict = None + ): + self.operation_id = operation_id + self.status_code = status_code self.method = method self.url = url self.params = params self.body = body self.headers = headers + self.global_auth = global_auth self.auth = auth @property @@ -61,7 +68,20 @@ def auth(self): @auth.setter def auth(self, value): - self._auth = Dict(value if value else {"username": "", "password": ""}) + if value: + self._auth = value + elif self.global_auth: + self._auth = Dict(self.global_auth) + else: + self._auth = {"username": "", "password": ""} + + @property + def global_auth(self): + return self._global_auth + + @global_auth.setter + def global_auth(self, value): + self._global_auth = value def evaluate_all(self) -> None: """ @@ -76,6 +96,6 @@ def evaluate_all(self) -> None: self.url = context.evaluate(self.url) # Evaluate all Dict object - for name in ("params", "body", "headers", "auth"): + for name in ("params", "body", "headers", "global_auth", "auth"): attr = self.__getattribute__(name) self.__setattr__(name, context.evaluate(attr)) diff --git a/src/models/response.py b/src/models/response.py index cf6ac94..06e04a8 100644 --- a/src/models/response.py +++ b/src/models/response.py @@ -20,7 +20,10 @@ class Response: def __getattr__(self, item): if item == "body": - body = self.__response.json() + if 'application/json' in self.__response.headers['Content-Type']: + body = self.__response.json() + else: + body = self.__response.content return Dict(body) if isinstance(body, dict) else body return getattr(self.__response, item) diff --git a/src/models/schema.py b/src/models/schema.py index 5867cb5..993b724 100644 --- a/src/models/schema.py +++ b/src/models/schema.py @@ -3,7 +3,6 @@ JSON Schema is a vocabulary that allows you to annotate and validate JSON documents. """ - from jschon import JSONSchema, JSON from jschon.jsonschema import Scope @@ -13,11 +12,21 @@ class Schema: """ Predefines meta schema and stores JSON schema document model. + See https://jschon.readthedocs.io/en/latest/reference/jsonschema.html#jschon.jsonschema.JSONSchema + See https://jschon.readthedocs.io/en/latest/examples/recursive_schema_extension.html for example """ def __init__(self, schema: dict, meta_schema: str = META_SCHEMA): - schema["$schema"] = meta_schema - self.schema = JSONSchema(schema) + json = {"$schema": meta_schema} + + # TODO: support more content types + try: + json.update(schema['content']['application/json']['schema']) + except KeyError as e: + print(e) + print('Key not found. Most likely content type of the response is not supported') + + self.schema = JSONSchema(json) def evaluate(self, json: dict) -> Scope: """ @@ -29,5 +38,5 @@ def evaluate(self, json: dict) -> Scope: Returns: Evaluate a JSON document and return the complete evaluation result tree. """ - + # inspect(self.schema, methods=True) return self.schema.evaluate(JSON(json)) diff --git a/src/models/singleton.py b/src/models/singleton.py index e6519ab..55e6110 100644 --- a/src/models/singleton.py +++ b/src/models/singleton.py @@ -4,6 +4,7 @@ This module is only used by Context to ensure the uniqueness of the global context state. """ + class Singleton(type): """ Restricts the instantiation of a class to one single instance. diff --git a/src/parsing.py b/src/parsing.py index 398ec72..fbad017 100644 --- a/src/parsing.py +++ b/src/parsing.py @@ -6,13 +6,10 @@ If the input file is not valid, it will raise an exception. """ -import importlib.resources -import json import os import yaml -from jschon import JSON, JSONSchema -from jschon.jsonschema import OutputFormat +from jsonschema import validate, ValidationError from openapi_spec_validator import validate_spec from prance import ResolvingParser, ValidationError from prance.util.url import ResolutionError @@ -20,8 +17,6 @@ from yaml.constructor import SafeConstructor from yaml.scanner import ScannerError as PyYAMLScannerError -from .errors import JSONValidatorError - def parse_spec(spec_file_path: str) -> dict: """Parse OpenAPI spec file to a dictionary. @@ -76,12 +71,10 @@ def parse_steps(step_file_path: str) -> dict: """ try: - with importlib.resources.path( - __package__, "steps_schema.json" - ) as step_schema_file: - steps_schema = JSONSchema(json.loads(step_schema_file.read_text())) + with open("./src/steps_schema.yml", "r") as step_schema_file: + steps_schema = yaml.safe_load(step_schema_file.read()) except FileNotFoundError as e: - raise FileNotFoundError(f"Steps schema file not found") from e + raise FileNotFoundError("Steps schema file not found") from e try: with open(step_file_path, "r") as step_file: @@ -92,11 +85,10 @@ def parse_steps(step_file_path: str) -> dict: # Load the step file and validate it against the step file schema yaml_dict = yaml.safe_load(step_file) - result = steps_schema.evaluate(JSON(yaml_dict)) - result_output = result.output(OutputFormat.BASIC) + validate(yaml_dict, steps_schema) - if not result_output["valid"]: - raise JSONValidatorError(result_output["errors"]) + except ValidationError as e: + raise ValidationError(f"Steps file has the following syntax errors: {e} ") from e except FileNotFoundError as e: raise FileNotFoundError(f"Steps file {step_file_path} not found") from e except PyYAMLScannerError as e: diff --git a/src/preprocessing.py b/src/preprocessing.py index 887ceeb..d0f16da 100644 --- a/src/preprocessing.py +++ b/src/preprocessing.py @@ -4,24 +4,24 @@ This module contains functions to ensure compatibility for a step file and corresponding spec file. It will check that every method referenced in the step file exists in the spec file. """ - +from rich import print from dataclasses import dataclass - +# TODO: refactor to `OperationCoverage` @dataclass -class EndpointCoverage: +class OperationCoverage: """ - Dictionary classifying endpoints into three categories: - - covered: endpoints that are called in the step file and documented in the spec file - - uncovered: endpoints that are in the spec file but not called in the step file - - undocumented: endpoints that are called in the step file but not documented in the spec file + Dictionary classifying operations into three categories: + - covered: operations that are called in the step file and documented in the spec file + - uncovered: operations that are in the spec file but not called in the step file + - undocumented: operations that are called in the step file but not documented in the spec file """ covered: set[str] uncovered: set[str] undocumented: set[str] - def has_undocumented_endpoints(self) -> bool: + def has_undocumented_operations(self) -> bool: return len(self.undocumented) > 0 def proportion_covered(self) -> float: @@ -30,26 +30,34 @@ def proportion_covered(self) -> float: ) -def get_endpoint_coverage(spec: dict, step: dict) -> EndpointCoverage: +# TODO: refactor to `get_operation_coverage` +def get_operation_coverage(spec: dict, step: dict) -> OperationCoverage: """ - Given a parsed spec and step file, determine the endpoints in the spec that are achieved by the step file. + Given a parsed spec and step file, determine the operations in the spec that are achieved by the step file. Args: spec (dict): specification file parsed as dict step (dict): step file parsed as dict Returns: - EndpointCoverage: A dataclass containing the endpoints covered, uncovered, and undocumented + OperationCoverage: A dataclass containing the operations covered, uncovered, and undocumented """ - - # Get all endpoints in the spec file - spec_endpoints = set(spec["paths"].keys()) - - # Get all endpoints in the step file - step_endpoints = set(step["paths"].keys()) - - return EndpointCoverage( - covered=spec_endpoints & step_endpoints, - uncovered=spec_endpoints - step_endpoints, - undocumented=step_endpoints - spec_endpoints, + print('---Checking Operation Coverage---') + + # Get all operations in the spec file + spec_operations = set() + for path in spec['paths']: + for method in spec['paths'][path]: + op_id = spec['paths'][path][method]['operationId'] + spec_operations.add(op_id) + + # Get all operations in the step file + step_operations = set() + for operation in step['operations']: + step_operations.add(operation['operation_id']) + + return OperationCoverage( + covered=spec_operations & step_operations, + uncovered=spec_operations - step_operations, + undocumented=step_operations - spec_operations, ) diff --git a/src/steps_schema.json b/src/steps_schema.json deleted file mode 100644 index f1b7c91..0000000 --- a/src/steps_schema.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$defs": { - "auth": { - "type": "object", - "properties": { - "username": { - "type": "string" - }, - "password": { - "type": "string" - } - }, - "required": ["username", "password"], - "additionalProperties": false - }, - "step": { - "type": "object", - "properties": { - "name": { - "type": "string", - "pattern": "\\w+" - }, - "method": { - "type": "string", - "pattern": "(?i)^(get|put|post|delete|patch)$" - }, - "url": { - "type": "string", - "pattern": "^(\/\\w*)+$" - }, - "headers": { - "type": "object" - }, - "body": { - "type": "object" - }, - "params": { - "type": "object" - }, - "auth": { - "$ref": "#/$defs/auth" - } - }, - "required": ["name", "method", "url"], - "additionalProperties": false - }, - "step-list": { - "type": "object", - "properties": { - "operationId": { - "type": "string" - }, - "steps": { - "type": "array", - "items": { - "$ref": "#/$defs/step" - } - } - }, - "required": ["operationId", "steps"], - "additionalProperties": false - }, - "path": { - "type": "object", - "patternProperties": { - "(?i)^(get|put|post|delete|patch)$": { - "$ref": "#/$defs/step-list" - } - }, - "additionalProperties": false - } - }, - "type": "object", - "properties": { - "base_url": { - "type": "string", - "format": "uri" - }, - "coverage_threshold": { - "type": "number", - "minimum": 0, - "maximum": 1 - }, - "paths": { - "type": "object", - "patternProperties": { - "^(\/(\\w|[{}])*)+$": { - "$ref": "#/$defs/path" - } - }, - "additionalProperties": false - } - }, - "required": ["base_url", "paths"], - "additionalProperties": false -} \ No newline at end of file diff --git a/src/steps_schema.yml b/src/steps_schema.yml new file mode 100644 index 0000000..0e28920 --- /dev/null +++ b/src/steps_schema.yml @@ -0,0 +1,68 @@ +--- +type: object +properties: + coverage_threshold: + type: number + minimum: 0 + maximum: 1 + base_url: + type: string + format: uri + auth: + "$ref": "#/$defs/auth" + operations: + type: array + items: + "$ref": "#/$defs/operation" + additionalProperties: false +required: +- base_url +- operations +additionalProperties: false + +"$defs": + auth: + type: object + properties: + username: + type: string + password: + type: string + required: + - username + - password + additionalProperties: false + operation: + type: object + properties: + name: + type: string + pattern: "\\w+" + operation_id: + type: string + pattern: "\\w+" + method: + type: string + pattern: "(?i)^(get|put|post|delete|patch)$" + url: + type: string + # pattern: "^(/\\w*)+$" TODO: Some URL validation here + headers: + type: object + body: + type: object + params: + type: object + auth: + "$ref": "#/$defs/auth" + status_code: + type: number + minimum: 100 + maximum: 600 + required: + - name + - operation_id + - method + - url + - status_code + additionalProperties: false diff --git a/src/validate.py b/src/validate.py index da38233..d3d6ab5 100644 --- a/src/validate.py +++ b/src/validate.py @@ -1,18 +1,15 @@ # -*- coding: utf-8 -*- """validate.py -This module contains the primary code for the GitHub Action. +This module contains the primary code for the package. Using the parsed OpenAPI specs and step files, it will create API requests and validate the responses. """ -import os import traceback import requests from rich import print, print_json -from dotenv import load_dotenv from jschon import create_catalog -from jschon.jsonschema import OutputFormat from openapi_schema_to_json_schema import to_json_schema from requests.auth import HTTPBasicAuth @@ -20,18 +17,15 @@ InsufficientCoverageError, ResponseMatchError, ResponseValidationError, - UndocumentedEndpointError, + UndocumentedOperationError, ) -from .models import Context, Request, Response, Schema, Step +from .models import Context, Request, Response, Schema, Operation from .parsing import parse_spec, parse_steps -from .preprocessing import get_endpoint_coverage +from .preprocessing import get_operation_coverage # Global variable storing all runtime contexts (initialize once) context = Context() -# Load dotenv -load_dotenv() - def parse_spec_steps(spec_file_path: str, step_file_path: str) -> tuple[dict, dict]: """ @@ -60,31 +54,32 @@ def parse_spec_steps(spec_file_path: str, step_file_path: str) -> tuple[dict, di return spec_data, steps_data -def check_endpoint_coverage(spec_data: dict, steps_data: dict): +def check_operation_coverage(spec_data: dict, steps_data: dict): """ - Checks the endpoint coverage of the step file against the OpenAPI spec. + Checks the operation coverage of the step file against the OpenAPI spec. Args: spec_data (dict): The parsed OpenAPI spec steps_data (dict): The parsed step file Raises: - UndocumentedEndpointError: + UndocumentedOperationError: """ - endpoint_coverage = get_endpoint_coverage(spec_data, steps_data) + operation_coverage = get_operation_coverage(spec_data, steps_data) + # inspect(operation_coverage) - # If any undocumented endpoints, immediately halt - if endpoint_coverage.has_undocumented_endpoints(): - raise UndocumentedEndpointError(endpoint_coverage.undocumented) + # If any undocumented operations, immediately halt + if operation_coverage.has_undocumented_operations(): + raise UndocumentedOperationError(operation_coverage.undocumented) - # Check if endpoint coverage meets threshold + # Check if operation coverage meets threshold if "coverage_threshold" in steps_data: target_coverage: float = steps_data["coverage_threshold"] - achieved_coverage = endpoint_coverage.proportion_covered() + achieved_coverage = operation_coverage.proportion_covered() if achieved_coverage < target_coverage: raise InsufficientCoverageError( - achieved_coverage, target_coverage, endpoint_coverage.uncovered + achieved_coverage, target_coverage, operation_coverage.uncovered ) @@ -104,113 +99,122 @@ def make_requests(spec_data: dict, steps_data: dict, fail_fast: bool, verbose: b print('---Validating APIs---') - # Create requests + # Set base url base_url: str = steps_data["base_url"] - paths: dict = steps_data["paths"] - - # Go through each path in the steps - path_value: dict - for path_key, path_value in paths.items(): - # Go through each method in each path - method_value: dict - for method_name, method_value in path_value.items(): - # Store steps of current method - context.clear_steps() - - # Get steps YAML from file - method_steps: list = method_value["steps"] - - # Go through each step in each method - step_data: dict - for step_data in method_steps: - try: - # Get step name - step_name = step_data.pop("name") - print(step_name) - - # Create Request object - path_url = step_data.pop("url") - request = Request(url=(base_url + path_url), **step_data) - - # Evaluate expressions - request.evaluate_all() - - print('Request:') - print(f'{request.method} {request.url}') - print(f'Authentication: {request.auth}') - print(f'Body:') - print_json(data=request.body, indent=4) - print(f'Headers: {request.headers}') - print(f'Parameters: {request.params}') - - # Create Step object - step = Step(step_name, request) - - # Send the request - step.response = Response( - requests.request( - method=request.method, - url=request.url, - params=request.params.to_dict(), - headers=request.headers.to_dict(), - json=request.body.to_dict(), - auth=HTTPBasicAuth(**request.auth), - ) - ) - - response = step.response - - print('Response:') - print(f'HTTP {response.status_code} {response.reason}') + + # Set global auth if it exists in the step file + global_auth: dict = {} + if 'auth' in steps_data.keys(): + global_auth: dict = steps_data["auth"] + + # Get operations list + operations: list = steps_data["operations"] + + # Create responses dict for easier parsing + operation_responses: dict = {} + for path in spec_data['paths']: + for method in spec_data['paths'][path]: + op_id = spec_data['paths'][path][method]['operationId'] + operation_responses[op_id] = spec_data['paths'][path][method]['responses'] + + # Go through each operation + operation_data: dict + for operation_data in operations: + try: + # Get operation name + operation_name = operation_data.pop("name") + + # Create Request object + path_url = operation_data.pop("url") + request = Request(url=(base_url + path_url), global_auth=global_auth, **operation_data) + + # TODO: something isnt right here - setting request body as application/xml + # in the spec but json in the step doesnt cause a failure + # Evaluate expressions + request.evaluate_all() + + # Create Operation object + operation = Operation(operation_name, request) + + # Send the request + operation.response = Response( + requests.request( + method=request.method, + url=request.url, + params=request.params.to_dict(), + headers=request.headers.to_dict(), + json=request.body.to_dict(), + auth=HTTPBasicAuth(**request.auth), + ) + ) + + response = operation.response + status_code = operation.response.status_code + + # Fetch schema + try: + schema = to_json_schema(operation_responses[operation_data['operation_id']][str(operation_data['status_code'])]) + except (AttributeError, KeyError): + raise ResponseMatchError( + operation_responses[operation_data['operation_id']].keys(), + operation.response, + ) + + # delete the $schema key that `to_json_schema` creates, it causes issues in the Schema class + try: + del schema['$schema'] + except KeyError: + pass + + operation.schema = Schema(schema) + + # Save the step to further use + context.add_operations(operation) + + # Verify the response + if 'application/json' in schema['content'].keys(): + verification_result = operation.verify() + else: + print('deez') + + # TODO: make this nicer using a rich table + if verbose: + print(f'Operation: {operation_name}') + print('--------------------') + print('Request:') + print(f'{request.method} {request.url}') + print(f'Authentication: {request.auth}') + print(f'Body:') + print_json(data=request.body, indent=4) + print(f'Headers: {request.headers}') + print(f'Parameters: {request.params}') + print('--------------------') + print('Response:') + print(f'HTTP {response.status_code} {response.reason}') + if type(response.body) == bytes: + print(f'Body size: {len(response.body)} bytes') + else: + print('Body:') print_json(data=response.body, indent=4) - print('') - - status_code = step.response.status_code - - # Fetch schema - try: - schema = to_json_schema( - spec_data.get("paths") - .get(path_key) - .get(method_name) - .get("responses") - .get(str(status_code)) - .get("content") - .get("application/json") - .get("schema") - ) - step.schema = Schema(schema) - except AttributeError: - raise ResponseMatchError( - spec_data.get("paths") - .get(path_key) - .get(method_name) - .get("responses") - .keys(), - step.response, - ) - - # Save the step to further use - context.add_steps(step) - - # Verify the response - verification_result = step.verify() - if not verification_result.valid: - raise ResponseValidationError( - errors=verification_result.output(OutputFormat.BASIC)["errors"], - url=path_url, - method=method_name, - status_code=status_code, - ) - - except BaseException as e: - if fail_fast: - raise e - - if verbose: - print(f'[red]{traceback.format_exc()}[/red]') - else: - print(f'[bold red]{str(e)}[/bold red]') + print('--------------------\n\n') + + + if not verification_result.valid: + raise ResponseValidationError( + errors=verification_result.output('basic')["errors"], + url=path_url, + method=operation_data['method'], + status_code=status_code, + ) + + except BaseException as e: + if fail_fast: + raise e + + if verbose: + print(f'[red]{traceback.format_exc()}[/red]') + else: + print(f'[bold red]{str(e)}[/bold red]') def verify_api(spec_file_path: str, step_file_path: str, fail_fast: bool = False, verbose: bool = False): @@ -226,13 +230,18 @@ def verify_api(spec_file_path: str, step_file_path: str, fail_fast: bool = False verbose (bool): Whether to output stacktrace """ - create_catalog("2020-12", default=True) + create_catalog("2020-12") # Parse spec and step files spec_data, steps_data = parse_spec_steps(spec_file_path, step_file_path) - # Check endpoint coverage - check_endpoint_coverage(spec_data, steps_data) + # TODO: This is broken. Outputs `paths' and exits + # the reason for this is that the original code looked for a `paths` dict in the spec and steps. + # with the overhaul of the schema to use operations, this logic was broken + # i think the best path forward is to compare operations instead of path coverage since paths can have multiple operations + # TODO: Refactor to `check_operation_coverage` + # Check operation coverage + check_operation_coverage(spec_data, steps_data) # Make requests make_requests(spec_data, steps_data, fail_fast, verbose) diff --git a/test/context_test.py b/test/context_test.py index f1573ea..a4e9f53 100644 --- a/test/context_test.py +++ b/test/context_test.py @@ -3,7 +3,7 @@ import pytest -from src.models import Context, Step, Request +from src.models import Context, Operation, Request from src.models.errors import BaseContextError, EnvironmentVariableError @@ -14,58 +14,78 @@ def context(): @pytest.fixture(autouse=True) def cleanup(context): - context.clear_steps() + context.clear_operations() yield - context.clear_steps() + context.clear_operations() def _env(value): return "${{ env." + value + " }}" -def _steps(value): - return "${{ steps." + value + " }}" +def _operations(value): + return "${{ operations." + value + " }}" def test_step_add(context): - step = Step( + operation = Operation( name="createUser", - request=Request(method="POST", url="http://localhost:8000/users"), + request=Request( + operation_id="createUser", + method="POST", + url="http://localhost:8000/users", + status_code=200 + ) ) - context.add_steps(step) + context.add_operations(operation) - assert context.steps.createUser == step + assert context.operations.createUser == operation def test_step_add_duplicated(context): for i in range(5): - context.add_steps( - Step( + context.add_operations( + Operation( name="createUser", - request=Request(method="POST", url="http://localhost:8000/users"), + request=Request( + operation_id="createUser", + method="POST", + url="http://localhost:8000/users", + status_code=200 + ) ) ) - assert len(vars(context.steps)) == 1 + assert len(vars(context.operations)) == 1 def test_step_clear(context): - context.add_steps( - Step( + context.add_operations( + Operation( name="createUser", - request=Request(method="POST", url="http://localhost:8000/users"), + request=Request( + operation_id="createUser", + method="POST", + url="http://localhost:8000/users", + status_code=200 + ) ) ) - context.add_steps( - Step( + context.add_operations( + Operation( name="getUsers", - request=Request(method="GET", url="http://localhost:8000/users"), + request=Request( + operation_id="getUsers", + method="GET", + url="http://localhost:8000/users", + status_code=200 + ) ) ) - assert len(vars(context.steps)) == 2 + assert len(vars(context.operations)) == 2 - context.clear_steps() - assert len(vars(context.steps)) == 0 + context.clear_operations() + assert len(vars(context.operations)) == 0 # Expression.evaluate() delegates Context.evaluate() @@ -102,21 +122,33 @@ def test_evaluate_multiple(context): def test_evaluate_url(context): user_id = str(uuid.uuid4()) account_id = str(uuid.uuid4()) - context.add_steps( - Step( - name="createUser", request=Request(body={"id": user_id}, url="", method="") + context.add_operations( + Operation( + name="createUser", + request=Request( + operation_id="createUser", + body={"id": user_id}, url="", + method="", + status_code=200 + ) ) ) - context.add_steps( - Step( + context.add_operations( + Operation( name="createAccount", - request=Request(body={"id": account_id}, url="", method=""), + request=Request( + operation_id="createAccount", + body={"id": account_id}, + url="", + method="", + status_code=200 + ) ) ) assert ( context.evaluate( - f"/users/{_steps('createUser.request.body.id')}/accounts/{_steps('createAccount.request.body.id')}" + f"/users/{_operations('createUser.request.body.id')}/accounts/{_operations('createAccount.request.body.id')}" ) == f"/users/{user_id}/accounts/{account_id}" ) diff --git a/test/fixtures/specs/valid/coverage_spec.yml b/test/fixtures/specs/valid/coverage_spec.yml index 948fc3f..17e23ae 100644 --- a/test/fixtures/specs/valid/coverage_spec.yml +++ b/test/fixtures/specs/valid/coverage_spec.yml @@ -7,10 +7,36 @@ info: It is meant to be used alongside coverage_steps.yml. version: 1.0.0 paths: - /covered_path: {} - /uncovered_path: {} + /covered_path: + get: + description: A covered operation + operationId: coveredOperation + responses: + "200": + description: Successful response + post: + description: A covered post operation + operationId: postOperation + responses: + "200": + description: Successful response + put: + description: A covered put operation + operationId: putOperation + responses: + "200": + description: Successful response + delete: + description: A covered delete operation + operationId: deleteOperation + responses: + "204": + description: Successful response + /uncovered_path: + get: + description: An uncovered operation + operationId: uncoveredOperation + responses: + "200": + description: Successful response # Steps will include an undocumented path - /nested/path: {} - /doubly/nested/path: {} - /path/{with}/braces: {} -components: {} diff --git a/test/fixtures/steps/invalid/scanner/invalid_steps1.yml b/test/fixtures/steps/invalid/scanner/invalid_steps1.yml deleted file mode 100644 index a8a2065..0000000 --- a/test/fixtures/steps/invalid/scanner/invalid_steps1.yml +++ /dev/null @@ -1,30 +0,0 @@ -# invalid YAML b/c paths is missing a : -base_url: "https://example.com" -paths - /person: - post: - operationId: - steps: - - name: createPerson - method: POST - headers: - Content-Type: application/json - data: - name: "John Doe" - age: 42 - get: - operationId: getPerson - steps: - - name: createPerson - method: POST - headers: - Content-Type: application/json - data: - name: "John Smith" - age: 30 - - name: getPerson - method: "GET" - url_parameters: - id: "{{ createPerson.response.id }}" - headers: - Content-Type: application/json diff --git a/test/fixtures/steps/invalid/scanner/invalid_yaml.yml b/test/fixtures/steps/invalid/scanner/invalid_yaml.yml new file mode 100644 index 0000000..1090a07 --- /dev/null +++ b/test/fixtures/steps/invalid/scanner/invalid_yaml.yml @@ -0,0 +1,28 @@ +base_url: "https://example.com" +# invalid yaml on line 2, operations is missing ':' +operations + - name: createPersonSuccessful + operation_id: createPerson + method: POST + url: /person + headers: + Content-Type: application/json + body: + name: John Doe + age: 42 + auth: + username: some_user + password: some_password + status_code: 200 + - name: fetchPersonInfoUnsSuccessful + operation_id: fetchPersonInfo + method: GET + url: /person + params: + id: ${{ operations.createPersonSuccessful.response.body.id }} + headers: + Content-Type: application/json + auth: + username: some_user + password: some_password + status_code: 200 \ No newline at end of file diff --git a/test/fixtures/steps/invalid/schema/invalid_steps2.yml b/test/fixtures/steps/invalid/schema/invalid_steps2.yml deleted file mode 100644 index 68e6923..0000000 --- a/test/fixtures/steps/invalid/schema/invalid_steps2.yml +++ /dev/null @@ -1,29 +0,0 @@ -base_url: "https://example.com" -# no operationId on the get step -paths: - /person: - post: - operationId: createPerson - steps: - - name: createPerson - method: POST - headers: - Content-Type: application/json - data: - name: "John Doe" - age: 42 - get: - steps: - - name: createPerson - method: POST - headers: - Content-Type: application/json - data: - name: "John Smith" - age: 30 - - name: getPerson - method: "GET" - url_parameters: - id: "{{ createPerson.response.id }}" - headers: - Content-Type: application/json diff --git a/test/fixtures/steps/invalid/schema/missing_property.yml b/test/fixtures/steps/invalid/schema/missing_property.yml new file mode 100644 index 0000000..5c6a6b2 --- /dev/null +++ b/test/fixtures/steps/invalid/schema/missing_property.yml @@ -0,0 +1,28 @@ +base_url: "https://example.com" +# no operation_id on the fetchPersonInfoUnsSuccessful operation +operations: + - name: createPersonSuccessful + operation_id: createPerson + method: POST + url: /person + headers: + Content-Type: application/json + body: + name: John Doe + age: 42 + auth: + username: some_user + password: some_password + status_code: 200 + - name: fetchPersonInfoUnsSuccessful + # operation_id: fetchPersonInfo + method: GET + url: /person + params: + id: ${{ operations.createPersonSuccessful.response.body.id }} + headers: + Content-Type: application/json + auth: + username: some_user + password: some_password + status_code: 200 \ No newline at end of file diff --git a/test/fixtures/steps/valid/coverage_steps.yml b/test/fixtures/steps/valid/coverage_steps.yml index 1e775f3..718ae21 100644 --- a/test/fixtures/steps/valid/coverage_steps.yml +++ b/test/fixtures/steps/valid/coverage_steps.yml @@ -1,9 +1,29 @@ coverage_threshold: 0.0 base_url: http://127.0.0.1:8000 -paths: - /covered_path: {} - # coverage includes an uncovered path - /undocumented_path: {} - /nested/path: {} - /doubly/nested/path: {} - /path/{with}/braces: {} +operations: + - name: coveredOperation + operation_id: coveredOperation + method: GET + url: / + status_code: 200 + # coverage intentionally does not include uncoveredOperation + - name: undocumentedOperation + operation_id: undocumentedOperation + method: GET + url: / + status_code: 200 + - name: postOperation + operation_id: postOperation + method: POST + url: / + status_code: 200 + - name: putOperation + operation_id: putOperation + method: PUT + url: / + status_code: 200 + - name: deleteOperation + operation_id: deleteOperation + method: GET + url: / + status_code: 204 diff --git a/test/fixtures/steps/valid/coverage_steps_all_documented.yml b/test/fixtures/steps/valid/coverage_steps_all_documented.yml index 0cb7a1d..9400c68 100644 --- a/test/fixtures/steps/valid/coverage_steps_all_documented.yml +++ b/test/fixtures/steps/valid/coverage_steps_all_documented.yml @@ -1,8 +1,24 @@ coverage_threshold: 0.99 base_url: http://127.0.0.1:8000 -paths: - /covered_path: {} - # coverage includes an uncovered path - /nested/path: {} - /doubly/nested/path: {} - /path/{with}/braces: {} +operations: + - name: coveredOperation + operation_id: coveredOperation + method: GET + url: / + status_code: 200 + # coverage intentionally does not include uncoveredOperation + - name: postOperation + operation_id: postOperation + method: POST + url: / + status_code: 200 + - name: putOperation + operation_id: putOperation + method: PUT + url: / + status_code: 200 + - name: deleteOperation + operation_id: deleteOperation + method: GET + url: / + status_code: 204 diff --git a/test/fixtures/steps/valid/jservice-multiple.steps.yml b/test/fixtures/steps/valid/jservice-multiple.steps.yml index f56e881..1db5586 100644 --- a/test/fixtures/steps/valid/jservice-multiple.steps.yml +++ b/test/fixtures/steps/valid/jservice-multiple.steps.yml @@ -1,18 +1,18 @@ base_url: https://jservice.io -paths: - /api/clues: - get: - operationId: getClues - steps: - - name: getAllClues1 - method: GET - url: /api/clues - params: - category: 4642 - value: 500 - - name: getAllClues2 - method: GET - url: /api/clues - params: - category: 4642 - value: 500 +operations: + - name: getAllCluesSuccessful1 + operation_id: getClues + method: GET + url: /api/clues + params: + category: 4642 + value: 500 + status_code: 200 + - name: getAllCluesSuccessful2 + operation_id: getClues + method: GET + url: /api/clues + params: + category: 4642 + value: 500 + status_code: 200 diff --git a/test/fixtures/steps/valid/jservice.steps.yml b/test/fixtures/steps/valid/jservice.steps.yml index 0fce5b9..5a3ff33 100644 --- a/test/fixtures/steps/valid/jservice.steps.yml +++ b/test/fixtures/steps/valid/jservice.steps.yml @@ -1,12 +1,10 @@ base_url: https://jservice.io -paths: - /api/clues: - get: - operationId: getClues - steps: - - name: getAllClues - method: GET - url: /api/clues - params: - category: 4642 - value: 500 +operations: + - name: getAllCluesSuccessful1 + operation_id: getClues + method: GET + url: /api/clues + params: + category: 4642 + value: 500 + status_code: 200 \ No newline at end of file diff --git a/test/fixtures/steps/valid/localhost.step.yml b/test/fixtures/steps/valid/localhost.step.yml index d5a3e10..73cb6bd 100644 --- a/test/fixtures/steps/valid/localhost.step.yml +++ b/test/fixtures/steps/valid/localhost.step.yml @@ -1,51 +1,49 @@ +coverage_threshold: 1.0 base_url: http://127.0.0.1:8000 -paths: - /: - get: - operationId: getEnv - steps: - - name: getIndex - method: GET - url: / - params: - environment: ${{ env.SERVER_ENV }} - /users: - post: - operationId: createUser - steps: - - name: createSampleUser - method: POST - url: /users - body: - first_name: First - last_name: Last - date_of_birth: 2000-01-01 - email: first.last@example.com - phone: 1234567890 - address: - number: 1234 - street: Example St. - city: Raleigh - state: NC - zip: 27607 - - name: createSampleUserClone - method: POST - url: /users - body: - first_name: ${{ steps.createSampleUser.response.body.first_name }}_clone - last_name: ${{ steps.createSampleUser.response.body.last_name }}_clone - date_of_birth: ${{ steps.createSampleUser.response.body.date_of_birth.replace('2000', '2001') }} - email: ${{ steps.createSampleUser.response.body.first_name }}_clone.${{ steps.createSampleUser.response.body.last_name }}_clone@example.com - phone: ${{ steps.createSampleUser.response.body.phone }} - address: - number: ${{ steps.createSampleUser.response.body.address.number }} - street: ${{ steps.createSampleUser.response.body.address.number }} - city: ${{ steps.createSampleUser.response.body.address.city }} - state: ${{ steps.createSampleUser.response.body.address.state }} - zip: ${{ steps.createSampleUser.response.body.address.zip }} - get: - operationId: getUser - steps: - - name: getAllUsers - method: GET - url: /users +operations: + - name: getEnvSuccessful + operation_id: getEnv + method: GET + url: / + params: + environment: ${{ env.SERVER_ENV }} + status_code: 200 + - name: createUserSuccessful + operation_id: createUser + method: POST + url: /users + body: + first_name: First + last_name: Last + date_of_birth: 2000-01-01 + email: first.last@example.com + phone: 1234567890 + address: + number: 1234 + street: Example St. + city: Raleigh + state: NC + zip: 27607 + status_code: 201 + - name: createSampleUserClone + operation_id: createUser + method: POST + url: /users + body: + first_name: ${{ steps.createSampleUser.response.body.first_name }}_clone + last_name: ${{ steps.createSampleUser.response.body.last_name }}_clone + date_of_birth: ${{ steps.createSampleUser.response.body.date_of_birth.replace('2000', '2001') }} + email: ${{ steps.createSampleUser.response.body.first_name }}_clone.${{ steps.createSampleUser.response.body.last_name }}_clone@example.com + phone: ${{ steps.createSampleUser.response.body.phone }} + address: + number: ${{ steps.createSampleUser.response.body.address.number }} + street: ${{ steps.createSampleUser.response.body.address.number }} + city: ${{ steps.createSampleUser.response.body.address.city }} + state: ${{ steps.createSampleUser.response.body.address.state }} + zip: ${{ steps.createSampleUser.response.body.address.zip }} + status_code: 201 + - name: getUsersSuccessful + operation_id: getUsers + method: GET + url: /users + status_code: 200 diff --git a/test/fixtures/steps/valid/localhost_uncovered.step.yml b/test/fixtures/steps/valid/localhost_uncovered.step.yml index 440cd3a..ccfb1fc 100644 --- a/test/fixtures/steps/valid/localhost_uncovered.step.yml +++ b/test/fixtures/steps/valid/localhost_uncovered.step.yml @@ -1,12 +1,10 @@ # only one method covered base_url: http://127.0.0.1:8000 -paths: - /: - get: - operationId: getEnv - steps: - - name: getIndex - method: GET - url: / - params: - environment: ${{ env.SERVER_ENV }} +operations: + - name: getEnvSuccessful + operation_id: getEnv + method: GET + url: / + params: + environment: ${{ env.SERVER_ENV }} + status_code: 200 diff --git a/test/fixtures/steps/valid/valid_steps1.yml b/test/fixtures/steps/valid/valid_steps1.yml index 3828b23..b698117 100644 --- a/test/fixtures/steps/valid/valid_steps1.yml +++ b/test/fixtures/steps/valid/valid_steps1.yml @@ -1,38 +1,27 @@ base_url: https://example.com -paths: - /person: - post: - operationId: createPerson - steps: - - name: createPerson - method: POST - url: /person - headers: - Content-Type: application/json - body: - name: John Doe - age: 42 - auth: - username: some_user - password: some_password - get: - operationId: fetchPersonInfo - steps: - - name: createPerson - method: POST - url: /person - headers: - Content-Type: application/json - body: - name: John Smith - age: 30 - - name: getPerson - method: GET - url: /person - params: - id: ${{ steps.createPerson.response.body.id }} - headers: - Content-Type: application/json - auth: - username: some_user - password: some_password +operations: + - name: createPersonSuccessful + operation_id: createPerson + method: POST + url: /person + headers: + Content-Type: application/json + body: + name: John Doe + age: 42 + auth: + username: some_user + password: some_password + status_code: 200 + - name: fetchPersonInfoSuccessful + operation_id: fetchPersonInfo + method: GET + url: /person + params: + id: ${{ operations.createPersonSuccessful.response.body.id }} + headers: + Content-Type: application/json + auth: + username: some_user + password: some_password + status_code: 200 diff --git a/test/parsing_test.py b/test/parsing_test.py index dcb14f5..4f9c0b2 100644 --- a/test/parsing_test.py +++ b/test/parsing_test.py @@ -9,7 +9,7 @@ def test_parse_valid_specs(valid_specs: list[str]): - required_keys = ["openapi", "info", "paths", "components"] + required_keys = ["openapi", "info", "paths"] for valid_spec in valid_specs: parsed_spec = parse_spec(valid_spec) assert parsed_spec.keys() & required_keys == set(required_keys) @@ -27,4 +27,4 @@ def test_parse_invalid_specs(invalid_specs: list[str]): def test_parse_valid_steps(valid_steps: list[str]): for valid_step in valid_steps: parsed_steps = parse_steps(valid_step) - assert {"base_url", "paths"}.issubset(parsed_steps.keys()) + assert {"base_url", "operations"}.issubset(parsed_steps.keys()) diff --git a/test/preprocessing_test.py b/test/preprocessing_test.py index 2fc15a3..1da925a 100644 --- a/test/preprocessing_test.py +++ b/test/preprocessing_test.py @@ -3,7 +3,7 @@ import os from src.parsing import parse_spec, parse_steps -from src.preprocessing import get_endpoint_coverage +from src.preprocessing import get_operation_coverage def test_compatible_spec_step(): @@ -19,8 +19,8 @@ def test_compatible_spec_step(): spec = parse_spec( os.path.join("test", "fixtures", "specs", "valid", "localhost.spec.yml") ) - endpoint_coverage = get_endpoint_coverage(spec, step) - assert not endpoint_coverage.has_undocumented_endpoints() + operation_coverage = get_operation_coverage(spec, step) + assert not operation_coverage.has_undocumented_operations() def test_missing_spec_method(): @@ -36,35 +36,35 @@ def test_missing_spec_method(): step = parse_steps( os.path.join("test", "fixtures", "steps", "valid", "localhost.step.yml") ) - endpoint_coverage = get_endpoint_coverage(spec, step) - assert endpoint_coverage.has_undocumented_endpoints() - assert "/" not in endpoint_coverage.undocumented + operation_coverage = get_operation_coverage(spec, step) + assert operation_coverage.has_undocumented_operations() + assert "/" not in operation_coverage.undocumented -def test_measure_endpoint_coverage(): +def test_measure_operation_coverage(): """ - Verify that endpoint coverage between specs and steps is measured correctly + Verify that operation coverage between specs and steps is measured correctly """ step = parse_steps("./test/fixtures/steps/valid/coverage_steps.yml") assert "coverage_threshold" in step spec = parse_spec("./test/fixtures/specs/valid/coverage_spec.yml") - endpoint_coverage = get_endpoint_coverage(spec, step) + operation_coverage = get_operation_coverage(spec, step) - assert endpoint_coverage.has_undocumented_endpoints() - assert len(endpoint_coverage.undocumented) == 1 - assert endpoint_coverage.undocumented == {"/undocumented_path"} + assert operation_coverage.has_undocumented_operations() + assert len(operation_coverage.undocumented) == 1 + assert operation_coverage.undocumented == {"undocumentedOperation"} - assert len(endpoint_coverage.covered) == 4 - assert endpoint_coverage.covered == { - "/covered_path", - "/nested/path", - "/doubly/nested/path", - "/path/{with}/braces", + assert len(operation_coverage.covered) == 4 + assert operation_coverage.covered == { + "coveredOperation", + "postOperation", + "putOperation", + "deleteOperation", } - assert len(endpoint_coverage.uncovered) == 1 - assert endpoint_coverage.uncovered == {"/uncovered_path"} + assert len(operation_coverage.uncovered) == 1 + assert operation_coverage.uncovered == {"uncoveredOperation"} - assert endpoint_coverage.proportion_covered() == 4 / (1 + 4 + 1) + assert operation_coverage.proportion_covered() == 4 / (1 + 4 + 1) diff --git a/test/request_test.py b/test/request_test.py index 81abfa8..2011f9b 100644 --- a/test/request_test.py +++ b/test/request_test.py @@ -1,10 +1,10 @@ -from src.models import Request, Context, Step +from src.models import Request, Context, Operation context = Context() def test_evaluate_all(): - step = Step( + operation = Operation( 'createUser', request=Request( method='POST', @@ -15,35 +15,39 @@ def test_evaluate_all(): { 'name': 'Hello' } - } + }, + operation_id='CreateUser', + status_code=200 ) ) - context.add_steps(step) + context.add_operations(operation) request = Request( method='GET', - url='${{ steps.createUser.request.url }}', + url='${{ operations.createUser.request.url }}', params={ 'user': { - 'token': '${{ steps.createUser.request.params.token }}' + 'token': '${{ operations.createUser.request.params.token }}' } }, body={ 'name': [ - '${{ steps.createUser.request.body.user.name }}', - '${{ steps.createUser.request.body.user.name }}' + '${{ operations.createUser.request.body.user.name }}', + '${{ operations.createUser.request.body.user.name }}' ] - } + }, + operation_id='GetUser', + status_code=200 ) request.evaluate_all() - assert request.url == step.request.url + assert request.url == operation.request.url # nested dict - assert request.params.user.token == step.request.params.token + assert request.params.user.token == operation.request.params.token # list for name in request.body.name: - assert name == step.request.body.user.name + assert name == operation.request.body.user.name diff --git a/test/step_test.py b/test/step_test.py index ff5ba71..a232f21 100644 --- a/test/step_test.py +++ b/test/step_test.py @@ -1,17 +1,19 @@ import json -from jschon import JSONSchema, create_catalog from src.errors import JSONValidatorError +from jsonschema import ValidationError +from jschon import create_catalog from src.parsing import parse_steps from yaml.scanner import ScannerError +from jschon import JSONSchema from .fixtures import * # Load JSON catalog and steps schema -create_catalog("2020-12", default=True) +create_catalog("2020-12") -with open("src/steps_schema.json", "r") as f: - schema = JSONSchema(json.load(f)) +# with open("src/steps_schema.yml", "r") as f: +# schema = JSONSchema(json.load(f)) def test_parse_valid_steps(valid_steps: list[str]): @@ -24,7 +26,7 @@ def test_parse_valid_steps(valid_steps: list[str]): def test_parse_invalid_steps(invalid_schema_steps: list[str]): for vs in invalid_schema_steps: - with pytest.raises(JSONValidatorError): + with pytest.raises(ValidationError): parse_steps(vs) diff --git a/test/validate_test.py b/test/validate_test.py index d8a5d72..7d65c8c 100644 --- a/test/validate_test.py +++ b/test/validate_test.py @@ -7,7 +7,7 @@ from src.validate import verify_api from src.errors import ( - UndocumentedEndpointError, + UndocumentedOperationError, InsufficientCoverageError, ResponseMatchError, ResponseValidationError, @@ -69,7 +69,7 @@ def test_step_scanner_error(): verify_api( os.path.join("test", "fixtures", "specs", "valid", "localhost.spec.yml"), os.path.join( - "test", "fixtures", "steps", "invalid", "scanner", "invalid_steps1.yml" + "test", "fixtures", "steps", "invalid", "scanner", "invalid_yaml.yml" ), ) @@ -81,8 +81,8 @@ def test_nullable_types(): ) -def test_undocumented_endpoints(): - with pytest.raises(UndocumentedEndpointError): +def test_undocumented_operations(): + with pytest.raises(UndocumentedOperationError): verify_api( os.path.join( "test", "fixtures", "specs", "valid", "localhost_undocumented.spec.yml" @@ -91,7 +91,7 @@ def test_undocumented_endpoints(): ) -def test_insufficient_endpoint_coverage(): +def test_insufficient_operation_coverage(): with pytest.raises(InsufficientCoverageError): verify_api( os.path.join("test", "fixtures", "specs", "valid", "coverage_spec.yml"), diff --git a/website/docs/user-guide.md b/website/docs/user-guide.md index b3cb828..31ef2f6 100644 --- a/website/docs/user-guide.md +++ b/website/docs/user-guide.md @@ -61,7 +61,7 @@ paths: At the top level of a steps file, there are two required fields: `base_url` and `paths`. The `base_url` parameter is the base URL to which all requests will be sent. Paths is an object and is explained below. -The `coverage_threshold` parameter is optional. If present, the action will compare the paths present in the spec to the paths present in the steps. If the proportion of paths present in the steps to paths present in the spec is less than the threshold, the action will fail and report which endpoints are uncovered. No matter if the `coverage_threshold` is present, the action will check for undocumented endpoints: those that are present in the steps but not the spec. If any of these are found, the action will fail. +The `coverage_threshold` parameter is optional. If present, the action will compare the paths present in the spec to the paths present in the steps. If the proportion of paths present in the steps to paths present in the spec is less than the threshold, the action will fail and report which operations are uncovered. No matter if the `coverage_threshold` is present, the action will check for undocumented operations: those that are present in the steps but not the spec. If any of these are found, the action will fail. Warning: The base_url should be a URL without a trailing slash and without query parameters. However, our schema enforces only that it must be a valid URI according to RFC3986.