From 12bf4d466337605737e100600d78c623740d5b41 Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Wed, 20 Apr 2022 19:27:48 -0400 Subject: [PATCH] SAT: support yaml files in spec_path (#12228) * support yaml files as spec_path * no unnecessary f-string * move to util for better testing * bump version, update changelog --- .../bases/source-acceptance-test/CHANGELOG.md | 3 ++ .../bases/source-acceptance-test/Dockerfile | 2 +- .../source_acceptance_test/conftest.py | 5 +-- .../source_acceptance_test/tests/test_core.py | 2 +- .../source_acceptance_test/utils/__init__.py | 3 +- .../source_acceptance_test/utils/common.py | 14 +++++++- .../unit_tests/test_utils.py | 36 +++++++++++++++++++ 7 files changed, 59 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md index e2e722da877a7..9301236d90ed6 100644 --- a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md +++ b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.1.50 +Added support for passing a `.yaml` file as `spec_path`. + ## 0.1.49 Fixed schema parsing when a JSONschema `type` was not present - we now assume `object` if the `type` is not present. diff --git a/airbyte-integrations/bases/source-acceptance-test/Dockerfile b/airbyte-integrations/bases/source-acceptance-test/Dockerfile index 94844e33e7c17..8ff75150da7f1 100644 --- a/airbyte-integrations/bases/source-acceptance-test/Dockerfile +++ b/airbyte-integrations/bases/source-acceptance-test/Dockerfile @@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./ COPY source_acceptance_test ./source_acceptance_test RUN pip install . -LABEL io.airbyte.version=0.1.49 +LABEL io.airbyte.version=0.1.50 LABEL io.airbyte.name=airbyte/source-acceptance-test ENTRYPOINT ["python", "-m", "pytest", "-p", "source_acceptance_test.plugin", "-r", "fEsx"] diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py index 52d046b7a0153..a86c3f2db0534 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py @@ -24,7 +24,7 @@ ) from docker import errors from source_acceptance_test.config import Config -from source_acceptance_test.utils import ConnectorRunner, SecretDict, load_config +from source_acceptance_test.utils import ConnectorRunner, SecretDict, load_config, load_yaml_or_json_path @pytest.fixture(name="base_path") @@ -117,7 +117,8 @@ def malformed_connector_config_fixture(connector_config) -> MutableMapping[str, @pytest.fixture(name="connector_spec") def connector_spec_fixture(connector_spec_path) -> ConnectorSpecification: - return ConnectorSpecification.parse_file(connector_spec_path) + spec_obj = load_yaml_or_json_path(connector_spec_path) + return ConnectorSpecification.parse_obj(spec_obj) @pytest.fixture(name="docker_runner") diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py index 0d07d8773b344..87b5afb9b3976 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py @@ -59,7 +59,7 @@ def test_config_match_spec(self, actual_connector_spec: ConnectorSpecification, def test_match_expected(self, connector_spec: ConnectorSpecification, actual_connector_spec: ConnectorSpecification): """Check that spec call returns a spec equals to expected one""" if connector_spec: - assert actual_connector_spec == connector_spec, "Spec should be equal to the one in spec.json file" + assert actual_connector_spec == connector_spec, "Spec should be equal to the one in spec.yaml or spec.json file" def test_docker_env(self, actual_connector_spec: ConnectorSpecification, docker_runner: ConnectorRunner): """Check that connector's docker image has required envs""" diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/__init__.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/__init__.py index e6267ca6aa5ac..618c814dfc96f 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/__init__.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/__init__.py @@ -2,7 +2,7 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # from .asserts import verify_records_schema -from .common import SecretDict, filter_output, full_refresh_only_catalog, incremental_only_catalog, load_config +from .common import SecretDict, filter_output, full_refresh_only_catalog, incremental_only_catalog, load_config, load_yaml_or_json_path from .compare import diff_dicts, make_hashable from .connector_runner import ConnectorRunner from .json_schema_helper import JsonSchemaHelper @@ -10,6 +10,7 @@ __all__ = [ "JsonSchemaHelper", "load_config", + "load_yaml_or_json_path", "filter_output", "full_refresh_only_catalog", "incremental_only_catalog", diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/common.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/common.py index 437d17a81390d..c00ed886922ff 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/common.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/common.py @@ -2,7 +2,7 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # - +import json from collections import UserDict from pathlib import Path from typing import Iterable, List, Union @@ -102,3 +102,15 @@ def _find_keyword(schema, key, _skip=False): except StopIteration: return True return False + + +def load_yaml_or_json_path(path: Path): + with open(str(path), "r") as file: + file_data = file.read() + file_ext = path.suffix + if file_ext == ".json": + return json.loads(file_data) + elif file_ext == ".yaml": + return load(file_data, Loader=Loader) + else: + raise RuntimeError("path must be a '.yaml' or '.json' file") diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_utils.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_utils.py index e8042c0ad9613..a39966a5a4fc3 100644 --- a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_utils.py +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_utils.py @@ -2,16 +2,21 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +import json import random import string +import tempfile import time from functools import partial +from pathlib import Path from typing import Iterable from unittest.mock import Mock import docker import pytest +import yaml from docker.errors import ContainerError, NotFound +from source_acceptance_test.utils.common import load_yaml_or_json_path from source_acceptance_test.utils.compare import make_hashable from source_acceptance_test.utils.connector_runner import ConnectorRunner @@ -310,3 +315,34 @@ def test_not_found_container(): with pytest.raises(NotFound): list(ConnectorRunner.read(new_container, command=cmd)) + + +class TestLoadYamlOrJsonPath: + VALID_SPEC = { + "documentationUrl": "https://google.com", + "connectionSpecification": { + "type": "object", + "required": ["api_token"], + "additionalProperties": False, + "properties": {"api_token": {"type": "string"}}, + }, + } + + def test_load_json(self): + with tempfile.NamedTemporaryFile("w", suffix=".json") as f: + f.write(json.dumps(self.VALID_SPEC)) + f.flush() + actual = load_yaml_or_json_path(Path(f.name)) + assert self.VALID_SPEC == actual + + def test_load_yaml(self): + with tempfile.NamedTemporaryFile("w", suffix=".yaml") as f: + f.write(yaml.dump(self.VALID_SPEC)) + f.flush() + actual = load_yaml_or_json_path(Path(f.name)) + assert self.VALID_SPEC == actual + + def test_load_other(self): + with tempfile.NamedTemporaryFile("w", suffix=".txt") as f: + with pytest.raises(RuntimeError): + load_yaml_or_json_path(Path(f.name))