CAT: improve test timeouts (#31810)

airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 2.1.1
+Centralize timeout duration declaration and increase timeout for discover.
+
 ## 2.1.0
 Make the container under test a sessions scoped fixture. 
 Support loading it from its Dagger container id for better performance. 

airbyte-integrations/bases/connector-acceptance-test/Dockerfile

@@ -24,7 +24,7 @@ RUN poetry install --no-root --only main --no-interaction --no-ansi
 COPY . /app
 RUN poetry install --only main --no-cache --no-interaction --no-ansi
 
-LABEL io.airbyte.version=2.0.2
+LABEL io.airbyte.version=2.1.1
 LABEL io.airbyte.name=airbyte/connector-acceptance-test
 WORKDIR /test_input
 ENTRYPOINT ["python", "-m", "pytest", "-p", "connector_acceptance_test.plugin", "-r", "fEsx", "--show-capture=log"]

airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py

@@ -54,6 +54,7 @@
     get_object_structure,
     get_paths_in_connector_config,
 )
+from connector_acceptance_test.utils.timeouts import FIVE_MINUTES, ONE_MINUTE, TEN_MINUTES
 
 pytestmark = [
     pytest.mark.anyio,
@@ -92,8 +93,9 @@ def secret_property_names_fixture():
 DATETIME_PATTERN = "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2})?$"
 
 
-# The connector fixture can be long to load, we have to increase the default timeout...
-@pytest.mark.default_timeout(5 * 60)
+# Running tests in parallel can sometime delay the execution of the tests if downstream services are not able to handle the load.
+# This is why we set a timeout on tests that call command that should return quickly, like spec
+@pytest.mark.default_timeout(ONE_MINUTE)
 class TestSpec(BaseTest):
     @pytest.fixture(name="skip_backward_compatibility_tests")
     async def skip_backward_compatibility_tests_fixture(
@@ -517,7 +519,7 @@ def test_oauth_flow_parameters(self, actual_connector_spec: ConnectorSpecificati
         diff = paths_to_validate - set(get_expected_schema_structure(spec_schema))
         assert diff == set(), f"Specified oauth fields are missed from spec schema: {diff}"
 
-    @pytest.mark.default_timeout(60)
+    @pytest.mark.default_timeout(ONE_MINUTE)
     @pytest.mark.backward_compatibility
     def test_backward_compatibility(
         self,
@@ -567,7 +569,7 @@ async def test_image_environment_variables(self, docker_runner: ConnectorRunner)
         assert await docker_runner.get_container_env_variable_value("AIRBYTE_ENTRYPOINT") == await docker_runner.get_container_entrypoint()
 
 
-@pytest.mark.default_timeout(30)
+@pytest.mark.default_timeout(ONE_MINUTE)
 class TestConnection(BaseTest):
     async def test_check(self, connector_config, inputs: ConnectionTestConfig, docker_runner: ConnectorRunner):
         if inputs.status == ConnectionTestConfig.Status.Succeed:
@@ -592,7 +594,9 @@ async def test_check(self, connector_config, inputs: ConnectionTestConfig, docke
             assert trace.error.message is not None
 
 
-@pytest.mark.default_timeout(30)
+# Running tests in parallel can sometime delay the execution of the tests if downstream services are not able to handle the load.
+# This is why we set a timeout on tests that call command that should return quickly, like discover
+@pytest.mark.default_timeout(FIVE_MINUTES)
 class TestDiscovery(BaseTest):
     VALID_TYPES = {"null", "string", "number", "integer", "boolean", "object", "array"}
     VALID_AIRBYTE_TYPES = {"timestamp_with_timezone", "timestamp_without_timezone", "integer"}
@@ -712,7 +716,7 @@ def test_additional_properties_is_true(self, discovered_catalog: Mapping[str, An
                     [additional_properties_value is True for additional_properties_value in additional_properties_values]
                 ), "When set, additionalProperties field value must be true for backward compatibility."
 
-    @pytest.mark.default_timeout(60)
+    @pytest.mark.default_timeout(ONE_MINUTE)
     @pytest.mark.backward_compatibility
     def test_backward_compatibility(
         self,
@@ -785,7 +789,7 @@ def primary_keys_for_records(streams, records):
             yield pk_values, stream_record
 
 
-@pytest.mark.default_timeout(10 * 60)
+@pytest.mark.default_timeout(TEN_MINUTES)
 class TestBasicRead(BaseTest):
     @staticmethod
     def _validate_records_structure(records: List[AirbyteRecordMessage], configured_catalog: ConfiguredAirbyteCatalog):

airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_full_refresh.py

@@ -14,8 +14,7 @@
 from connector_acceptance_test.config import IgnoredFieldsConfiguration
 from connector_acceptance_test.utils import ConnectorRunner, JsonSchemaHelper, SecretDict, full_refresh_only_catalog, make_hashable
 from connector_acceptance_test.utils.json_schema_helper import CatalogField
-
-# from airbyte_pr import ConfiguredAirbyteCatalog, Type
+from connector_acceptance_test.utils.timeouts import TWENTY_MINUTES
 
 
 def primary_keys_by_stream(configured_catalog: ConfiguredAirbyteCatalog) -> Mapping[str, List[CatalogField]]:
@@ -37,7 +36,7 @@ def primary_keys_only(record, pks):
     return ";".join([f"{pk.path}={pk.parse(record)}" for pk in pks])
 
 
-@pytest.mark.default_timeout(20 * 60)
+@pytest.mark.default_timeout(TWENTY_MINUTES)
 class TestFullRefresh(BaseTest):
     def assert_emitted_at_increase_on_subsequent_runs(self, first_read_records, second_read_records):
         first_read_records_data = [record.data for record in first_read_records]

airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_incremental.py

@@ -3,17 +3,15 @@
 #
 
 import json
-from collections import defaultdict
-from datetime import datetime
 from pathlib import Path
-from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Tuple, Union
+from typing import Any, Dict, List, Mapping, MutableMapping, Tuple, Union
 
-import pendulum
 import pytest
 from airbyte_protocol.models import AirbyteMessage, AirbyteStateMessage, AirbyteStateType, ConfiguredAirbyteCatalog, SyncMode, Type
 from connector_acceptance_test import BaseTest
 from connector_acceptance_test.config import Config, EmptyStreamConfiguration, IncrementalConfig
-from connector_acceptance_test.utils import ConnectorRunner, JsonSchemaHelper, SecretDict, filter_output, incremental_only_catalog
+from connector_acceptance_test.utils import ConnectorRunner, SecretDict, filter_output, incremental_only_catalog
+from connector_acceptance_test.utils.timeouts import TWENTY_MINUTES
 from deepdiff import DeepDiff
 
 
@@ -103,7 +101,7 @@ def naive_diff_records(records_1: List[AirbyteMessage], records_2: List[AirbyteM
     return diff
 
 
-@pytest.mark.default_timeout(20 * 60)
+@pytest.mark.default_timeout(TWENTY_MINUTES)
 class TestIncremental(BaseTest):
     async def test_two_sequential_reads(
         self,

airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/timeouts.py

@@ -0,0 +1,7 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+ONE_MINUTE = 60
+FIVE_MINUTES = 60 * 5
+TEN_MINUTES = 60 * 10
+TWENTY_MINUTES = 60 * 20

airbyte-integrations/bases/connector-acceptance-test/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "connector-acceptance-test"
-version = "2.1.0"
+version = "2.1.1"
 description = "Contains acceptance tests for connectors."
 authors = ["Airbyte <contact@airbyte.io>"]
 license = "MIT"