refactor: Reduce tests flakiness

src/schemathesis/converter.py

@@ -1,8 +1,10 @@
+from copy import deepcopy
 from typing import Any, Dict
 
 
 def to_json_schema(schema: Dict[str, Any], nullable_name: str) -> Dict[str, Any]:
     """Convert Open API parameters to JSON Schema."""
+    schema = deepcopy(schema)
     if schema.get(nullable_name) is True:
         del schema[nullable_name]
         if schema.get("in"):

test/cli/test_callbacks.py

@@ -22,7 +22,7 @@ def test_validate_schema_path_without_base_url():
         callbacks.validate_schema(SimpleNamespace(params={}), None, SIMPLE_PATH)
 
 
-@given(value=st.text().filter(lambda x: not x.endswith(":")))
+@given(value=st.text().filter(lambda x: x.count(":") != 1))
 @example(":")
 def test_validate_auth(value):
     with pytest.raises(click.BadParameter):

test/cli/test_commands.py

@@ -352,7 +352,7 @@ def test_cli_run_only_failure(cli, cli_args, workers):
 
 @pytest.mark.endpoints("upload_file")
 def test_cli_binary_body(cli, schema_url):
-    result = cli.run(schema_url)
+    result = cli.run(schema_url, "--hypothesis-suppress-health-check=filter_too_much")
     assert result.exit_code == ExitCode.OK
     assert " HYPOTHESIS OUTPUT " not in result.stdout
 
@@ -661,7 +661,9 @@ def test_pre_run_hook_valid(testdir, cli, schema_url, app):
     """
     )
 
-    result = cli.main("--pre-run", module.purebasename, "run", schema_url)
+    result = cli.main(
+        "--pre-run", module.purebasename, "run", "--hypothesis-suppress-health-check=filter_too_much", schema_url
+    )
 
     # Then CLI should run successfully
     assert result.exit_code == ExitCode.OK

test/test_hypothesis.py

@@ -1,7 +1,7 @@
 from base64 import b64decode
 
 import pytest
-from hypothesis import given, settings, strategies
+from hypothesis import HealthCheck, given, settings, strategies
 
 from schemathesis import Case, register_string_format
 from schemathesis._hypothesis import PARAMETERS, get_case_strategy, get_examples, is_valid_query
@@ -156,6 +156,7 @@ def test_valid_headers(base_url, swagger_20):
     )
 
     @given(case=get_case_strategy(endpoint))
+    @settings(suppress_health_check=[HealthCheck.filter_too_much])
     def inner(case):
         case.call()
 

test/test_parameters.py

@@ -1,7 +1,7 @@
 import datetime
 
 import yaml
-from hypothesis import given
+from hypothesis import HealthCheck, given, settings
 
 import schemathesis
 
@@ -13,6 +13,7 @@ def test_headers(testdir):
     testdir.make_test(
         """
 @schema.parametrize()
+@settings(suppress_health_check=[HealthCheck.filter_too_much])
 def test_(case):
     assert_str(case.headers["api_key"])
     assert_requests_call(case)
@@ -28,6 +29,7 @@ def test_cookies(testdir):
     testdir.make_test(
         """
 @schema.parametrize()
+@settings(suppress_health_check=[HealthCheck.filter_too_much])
 def test_(case):
     assert_str(case.cookies["token"])
     assert_requests_call(case)
@@ -153,6 +155,7 @@ def test_date_deserializing(testdir):
     schema = schemathesis.from_path(str(schema_path))
 
     @given(case=schema["/teapot"]["GET"].as_strategy())
+    @settings(suppress_health_check=[HealthCheck.filter_too_much])
     def test(case):
         assert isinstance(case.query["key"], str)
 

test/test_petstore.py

@@ -39,7 +39,7 @@ def test_find_by_status(testdir):
     testdir.make_petstore_test(
         """
 @schema.parametrize(endpoint="/pet/findByStatus$")
-@settings(max_examples=5)
+@settings(max_examples=5, deadline=None)
 def test_(request, case):
     request.config.HYPOTHESIS_CASES += 1
     assert_list(case.query["status"])
@@ -83,7 +83,7 @@ def test_update_pet(testdir):
     testdir.make_petstore_test(
         """
 @schema.parametrize(method="POST", endpoint="/pet/{petId}$")
-@settings(max_examples=5)
+@settings(max_examples=5, deadline=None)
 def test_(request, case):
     request.config.HYPOTHESIS_CASES += 1
     assert_int(case.path_parameters["petId"])
@@ -176,7 +176,7 @@ def test_delete_order(testdir):
     testdir.make_petstore_test(
         """
 @schema.parametrize(method="DELETE", endpoint="/store/order/{orderId}$")
-@settings(max_examples=5)
+@settings(max_examples=5, suppress_health_check=[HealthCheck.filter_too_much])
 def test_(request, case):
     request.config.HYPOTHESIS_CASES += 1
     assert_int(case.path_parameters["orderId"])
@@ -191,7 +191,7 @@ def test_create_user(testdir):
     testdir.make_petstore_test(
         """
 @schema.parametrize(endpoint="/user$")
-@settings(max_examples=5)
+@settings(max_examples=5, deadline=None)
 def test_(request, case):
     request.config.HYPOTHESIS_CASES += 1
     assert isinstance(case.body, dict)
@@ -205,7 +205,7 @@ def test_create_multiple_users(testdir):
     testdir.make_petstore_test(
         """
 @schema.parametrize(endpoint="/user/createWith")
-@settings(max_examples=5)
+@settings(max_examples=5, deadline=None)
 def test_(request, case):
     request.config.HYPOTHESIS_CASES += 1
     assert_list(case.body)

test/test_required.py

@@ -5,7 +5,7 @@ def test_required_parameters(testdir):
     testdir.make_test(
         """
 @schema.parametrize(method="POST")
-@settings(max_examples=20)
+@settings(max_examples=20, suppress_health_check=[HealthCheck.data_too_large])
 def test_(request, case):
     request.config.HYPOTHESIS_CASES += 1
     assert case.path == "/v1/users"

test/test_schemas.py

@@ -43,14 +43,15 @@ def test_open_api_verbose_name(openapi_30):
     assert openapi_30.spec_version == "3.0.0"
 
 
-def test_resolver_cache(swagger_20, mocker):
+def test_resolver_cache(simple_schema, mocker):
+    schema = schemathesis.from_dict(simple_schema)
     spy = mocker.patch("schemathesis.schemas.jsonschema.RefResolver", wraps=RefResolver)
-    assert "_resolver" not in swagger_20.__dict__
-    assert isinstance(swagger_20.resolver, RefResolver)
+    assert "_resolver" not in schema.__dict__
+    assert isinstance(schema.resolver, RefResolver)
     assert spy.call_count == 1
     # Cached
-    assert "_resolver" in swagger_20.__dict__
-    assert isinstance(swagger_20.resolver, RefResolver)
+    assert "_resolver" in schema.__dict__
+    assert isinstance(schema.resolver, RefResolver)
     assert spy.call_count == 1