SAT: support yaml files in spec_path (#12228)

* support yaml files as spec_path

* no unnecessary f-string

* move to util for better testing

* bump version, update changelog

airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.1.50
+Added support for passing a `.yaml` file as `spec_path`.
+
 ## 0.1.49
 Fixed schema parsing when a JSONschema `type` was not present - we now assume `object` if the `type` is not present.
 

airbyte-integrations/bases/source-acceptance-test/Dockerfile

@@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./
 COPY source_acceptance_test ./source_acceptance_test
 RUN pip install .
 
-LABEL io.airbyte.version=0.1.49
+LABEL io.airbyte.version=0.1.50
 LABEL io.airbyte.name=airbyte/source-acceptance-test
 
 ENTRYPOINT ["python", "-m", "pytest", "-p", "source_acceptance_test.plugin", "-r", "fEsx"]

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py

@@ -24,7 +24,7 @@
 )
 from docker import errors
 from source_acceptance_test.config import Config
-from source_acceptance_test.utils import ConnectorRunner, SecretDict, load_config
+from source_acceptance_test.utils import ConnectorRunner, SecretDict, load_config, load_yaml_or_json_path
 
 
 @pytest.fixture(name="base_path")
@@ -117,7 +117,8 @@ def malformed_connector_config_fixture(connector_config) -> MutableMapping[str,
 
 @pytest.fixture(name="connector_spec")
 def connector_spec_fixture(connector_spec_path) -> ConnectorSpecification:
-    return ConnectorSpecification.parse_file(connector_spec_path)
+    spec_obj = load_yaml_or_json_path(connector_spec_path)
+    return ConnectorSpecification.parse_obj(spec_obj)
 
 
 @pytest.fixture(name="docker_runner")

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py

@@ -59,7 +59,7 @@ def test_config_match_spec(self, actual_connector_spec: ConnectorSpecification,
     def test_match_expected(self, connector_spec: ConnectorSpecification, actual_connector_spec: ConnectorSpecification):
         """Check that spec call returns a spec equals to expected one"""
         if connector_spec:
-            assert actual_connector_spec == connector_spec, "Spec should be equal to the one in spec.json file"
+            assert actual_connector_spec == connector_spec, "Spec should be equal to the one in spec.yaml or spec.json file"
 
     def test_docker_env(self, actual_connector_spec: ConnectorSpecification, docker_runner: ConnectorRunner):
         """Check that connector's docker image has required envs"""

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/__init__.py

@@ -2,14 +2,15 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 from .asserts import verify_records_schema
-from .common import SecretDict, filter_output, full_refresh_only_catalog, incremental_only_catalog, load_config
+from .common import SecretDict, filter_output, full_refresh_only_catalog, incremental_only_catalog, load_config, load_yaml_or_json_path
 from .compare import diff_dicts, make_hashable
 from .connector_runner import ConnectorRunner
 from .json_schema_helper import JsonSchemaHelper
 
 __all__ = [
     "JsonSchemaHelper",
     "load_config",
+    "load_yaml_or_json_path",
     "filter_output",
     "full_refresh_only_catalog",
     "incremental_only_catalog",

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/common.py

@@ -2,7 +2,7 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
-
+import json
 from collections import UserDict
 from pathlib import Path
 from typing import Iterable, List, Union
@@ -102,3 +102,15 @@ def _find_keyword(schema, key, _skip=False):
     except StopIteration:
         return True
     return False
+
+
+def load_yaml_or_json_path(path: Path):
+    with open(str(path), "r") as file:
+        file_data = file.read()
+        file_ext = path.suffix
+        if file_ext == ".json":
+            return json.loads(file_data)
+        elif file_ext == ".yaml":
+            return load(file_data, Loader=Loader)
+        else:
+            raise RuntimeError("path must be a '.yaml' or '.json' file")

airbyte-integrations/bases/source-acceptance-test/unit_tests/test_utils.py

@@ -2,16 +2,21 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
+import json
 import random
 import string
+import tempfile
 import time
 from functools import partial
+from pathlib import Path
 from typing import Iterable
 from unittest.mock import Mock
 
 import docker
 import pytest
+import yaml
 from docker.errors import ContainerError, NotFound
+from source_acceptance_test.utils.common import load_yaml_or_json_path
 from source_acceptance_test.utils.compare import make_hashable
 from source_acceptance_test.utils.connector_runner import ConnectorRunner
 
@@ -310,3 +315,34 @@ def test_not_found_container():
 
     with pytest.raises(NotFound):
         list(ConnectorRunner.read(new_container, command=cmd))
+
+
+class TestLoadYamlOrJsonPath:
+    VALID_SPEC = {
+        "documentationUrl": "https://google.com",
+        "connectionSpecification": {
+            "type": "object",
+            "required": ["api_token"],
+            "additionalProperties": False,
+            "properties": {"api_token": {"type": "string"}},
+        },
+    }
+
+    def test_load_json(self):
+        with tempfile.NamedTemporaryFile("w", suffix=".json") as f:
+            f.write(json.dumps(self.VALID_SPEC))
+            f.flush()
+            actual = load_yaml_or_json_path(Path(f.name))
+            assert self.VALID_SPEC == actual
+
+    def test_load_yaml(self):
+        with tempfile.NamedTemporaryFile("w", suffix=".yaml") as f:
+            f.write(yaml.dump(self.VALID_SPEC))
+            f.flush()
+            actual = load_yaml_or_json_path(Path(f.name))
+            assert self.VALID_SPEC == actual
+
+    def test_load_other(self):
+        with tempfile.NamedTemporaryFile("w", suffix=".txt") as f:
+            with pytest.raises(RuntimeError):
+                load_yaml_or_json_path(Path(f.name))