diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index a1ce83a8a2c36..4612a87fa4854 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.7.0 +Low-code: Allow connector specifications to be defined in the manifest + ## 0.6.0 Low-code: Add support for monthly and yearly incremental updates for `DatetimeStreamSlicer` diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py index 6f18376f2a85b..66d4d6b0bc905 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py @@ -33,6 +33,7 @@ from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import PageIncrement from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever from airbyte_cdk.sources.declarative.schema.json_file_schema_loader import JsonFileSchemaLoader +from airbyte_cdk.sources.declarative.spec import Spec from airbyte_cdk.sources.declarative.stream_slicers.cartesian_product_stream_slicer import CartesianProductStreamSlicer from airbyte_cdk.sources.declarative.stream_slicers.datetime_stream_slicer import DatetimeStreamSlicer from airbyte_cdk.sources.declarative.stream_slicers.list_stream_slicer import ListStreamSlicer @@ -75,6 +76,7 @@ "RemoveFields": RemoveFields, "SimpleRetriever": SimpleRetriever, "SingleSlice": SingleSlice, + "Spec": Spec, "SubstreamSlicer": SubstreamSlicer, "WaitUntilTimeFromHeader": WaitUntilTimeFromHeaderBackoffStrategy, "WaitTimeFromHeader": WaitTimeFromHeaderBackoffStrategy, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py index b499ea756132f..0904fb3bb66a3 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py @@ -246,7 +246,11 @@ def is_object_definition_with_class_name(definition): @staticmethod def is_object_definition_with_type(definition): - return isinstance(definition, dict) and "type" in definition + # The `type` field is an overloaded term in the context of the low-code manifest. As part of the language, `type` is shorthand + # for convenience to avoid defining the entire classpath. For the connector specification, `type` is a part of the spec schema. + # For spec parsing, as part of this check, when the type is set to object, we want it to remain a mapping. But when type is + # defined any other way, then it should be parsed as a declarative component in the manifest. + return isinstance(definition, dict) and "type" in definition and definition["type"] != "object" @staticmethod def get_default_type(parameter_name, parent_class): diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/__init__.py new file mode 100644 index 0000000000000..fba2f9612ba2e --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/__init__.py @@ -0,0 +1,7 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.spec.spec import Spec + +__all__ = ["Spec"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py new file mode 100644 index 0000000000000..d5ac6a1d586d3 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from dataclasses import InitVar, dataclass +from typing import Any, Mapping + +from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification +from dataclasses_jsonschema import JsonSchemaMixin + + +@dataclass +class Spec(JsonSchemaMixin): + """ + Returns a connection specification made up of information about the connector and how it can be configured + + Attributes: + documentation_url (str): The link the Airbyte documentation about this connector + connection_specification (Mapping[str, Any]): information related to how a connector can be configured + """ + + documentation_url: str + connection_specification: Mapping[str, Any] + options: InitVar[Mapping[str, Any]] + + def generate_spec(self) -> ConnectorSpecification: + """ + Returns the connector specification according the spec block defined in the low code connector manifest. + """ + + # We remap these keys to camel case because that's the existing format expected by the rest of the platform + return ConnectorSpecification.parse_obj( + {"documentationUrl": self.documentation_url, "connectionSpecification": self.connection_specification} + ) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py index 9fdffcc703f4e..181ddc096d99a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py @@ -5,7 +5,6 @@ import datetime import re from dataclasses import InitVar, dataclass, field -from dateutil.relativedelta import relativedelta from typing import Any, Iterable, Mapping, Optional, Union from airbyte_cdk.models import SyncMode @@ -17,6 +16,7 @@ from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState from dataclasses_jsonschema import JsonSchemaMixin +from dateutil.relativedelta import relativedelta @dataclass @@ -71,7 +71,9 @@ class DatetimeStreamSlicer(StreamSlicer, JsonSchemaMixin): stream_state_field_end: Optional[str] = None lookback_window: Optional[Union[InterpolatedString, str]] = None - timedelta_regex = re.compile(r"((?P[\.\d]+?)y)?" r"((?P[\.\d]+?)m)?" r"((?P[\.\d]+?)w)?" r"((?P[\.\d]+?)d)?$") + timedelta_regex = re.compile( + r"((?P[\.\d]+?)y)?" r"((?P[\.\d]+?)m)?" r"((?P[\.\d]+?)w)?" r"((?P[\.\d]+?)d)?$" + ) def __post_init__(self, options: Mapping[str, Any]): if not isinstance(self.start_datetime, MinMaxDatetime): diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py index c9e31fd47ba92..440b8cc5c9823 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py @@ -11,6 +11,7 @@ from enum import Enum, EnumMeta from typing import Any, List, Mapping, Union +from airbyte_cdk.models import ConnectorSpecification from airbyte_cdk.sources.declarative.checks import CheckStream from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource @@ -33,7 +34,7 @@ class ConcreteDeclarativeSource(JsonSchemaMixin): class YamlDeclarativeSource(DeclarativeSource): """Declarative source defined by a yaml file""" - VALID_TOP_LEVEL_FIELDS = {"definitions", "streams", "check", "version"} + VALID_TOP_LEVEL_FIELDS = {"check", "definitions", "spec", "streams", "version"} def __init__(self, path_to_yaml): """ @@ -69,6 +70,28 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: self._apply_log_level_to_stream_logger(self.logger, stream) return source_streams + def spec(self, logger: logging.Logger) -> ConnectorSpecification: + """ + Returns the connector specification (spec) as defined in the Airbyte Protocol. The spec is an object describing the possible + configurations (e.g: username and password) which can be configured when running this connector. For low-code connectors, this + will first attempt to load the spec from the manifest's spec block, otherwise it will load it from "spec.yaml" or "spec.json" + in the project root. + """ + + self.logger.debug( + "parsed YAML into declarative source", + extra={"path_to_yaml_file": self._path_to_yaml, "source_name": self.name, "parsed_config": json.dumps(self._source_config)}, + ) + + spec = self._source_config.get("spec") + if spec: + if "class_name" not in spec: + spec["class_name"] = "airbyte_cdk.sources.declarative.spec.Spec" + spec_component = self._factory.create_component(spec, dict())() + return spec_component.generate_spec() + else: + return super().spec(logger) + def _read_and_parse_yaml_file(self, path_to_yaml_file): package = self.__class__.__module__.split(".")[0] diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 02c854cb6e1a9..46a84500bc661 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.6.0", + version="0.7.0", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py index a187a4eb5d358..88ad065ac6909 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py @@ -3,7 +3,6 @@ # import datetime -from dateutil.relativedelta import relativedelta from typing import List, Optional, Union import pytest @@ -41,6 +40,7 @@ from airbyte_cdk.sources.declarative.stream_slicers.list_stream_slicer import ListStreamSlicer from airbyte_cdk.sources.declarative.transformations import AddFields, RemoveFields from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition +from dateutil.relativedelta import relativedelta from jsonschema import ValidationError factory = DeclarativeComponentFactory() @@ -345,6 +345,22 @@ def test_full_config(): check: class_name: airbyte_cdk.sources.declarative.checks.check_stream.CheckStream stream_names: ["list_stream"] +spec: + class_name: airbyte_cdk.sources.declarative.spec.Spec + documentation_url: https://airbyte.com/#yaml-from-manifest + connection_specification: + title: Test Spec + type: object + required: + - api_key + additionalProperties: false + properties: + api_key: + type: string + airbyte_secret: true + title: API Key + description: Test API Key + order: 0 """ config = parser.parse(content) @@ -377,6 +393,20 @@ def test_full_config(): assert len(streams_to_check) == 1 assert list(streams_to_check)[0] == "list_stream" + spec = factory.create_component(config["spec"], input_config)() + documentation_url = spec.documentation_url + connection_specification = spec.connection_specification + assert documentation_url == "https://airbyte.com/#yaml-from-manifest" + assert connection_specification["title"] == "Test Spec" + assert connection_specification["required"] == ["api_key"] + assert connection_specification["properties"]["api_key"] == { + "type": "string", + "airbyte_secret": True, + "title": "API Key", + "description": "Test API Key", + "order": 0, + } + assert stream.retriever.requester.path.default == "marketing/lists" diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py index 340d765a52ffb..adceb433191e3 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py @@ -3,252 +3,466 @@ # import json +import logging +import os +import sys +import tempfile +import pytest +import yaml +from airbyte_cdk.sources.declarative.exceptions import InvalidConnectorDefinitionException +from airbyte_cdk.sources.declarative.parsers.yaml_parser import YamlParser from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource +from jsonschema import ValidationError -# import pytest -# from airbyte_cdk.sources.declarative.exceptions import InvalidConnectorDefinitionException - -# import os -# import tempfile -# import unittest - - -# from jsonschema import ValidationError - - -# brianjlai: Commenting these out for the moment because I can't figure out why the temp file is unreadable at runtime during testing -# its more urgent to fix the connectors -# class TestYamlDeclarativeSource(unittest.TestCase): -# def test_source_is_created_if_toplevel_fields_are_known(self): -# content = """ -# version: "version" -# definitions: -# schema_loader: -# name: "{{ options.stream_name }}" -# file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" -# retriever: -# paginator: -# type: "DefaultPaginator" -# page_size: 10 -# page_size_option: -# inject_into: request_parameter -# field_name: page_size -# page_token_option: -# inject_into: path -# pagination_strategy: -# type: "CursorPagination" -# cursor_value: "{{ response._metadata.next }}" -# requester: -# path: "/v3/marketing/lists" -# authenticator: -# type: "BearerAuthenticator" -# api_token: "{{ config.apikey }}" -# request_parameters: -# page_size: 10 -# record_selector: -# extractor: -# field_pointer: ["result"] -# streams: -# - type: DeclarativeStream -# $options: -# name: "lists" -# primary_key: id -# url_base: "https://api.sendgrid.com" -# schema_loader: "*ref(definitions.schema_loader)" -# retriever: "*ref(definitions.retriever)" -# check: -# type: CheckStream -# stream_names: ["lists"] -# """ -# temporary_file = TestFileContent(content) -# YamlDeclarativeSource(temporary_file.filename) -# -# def test_source_is_not_created_if_toplevel_fields_are_unknown(self): -# content = """ -# version: "version" -# definitions: -# schema_loader: -# name: "{{ options.stream_name }}" -# file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" -# retriever: -# paginator: -# type: "DefaultPaginator" -# page_size: 10 -# page_size_option: -# inject_into: request_parameter -# field_name: page_size -# page_token_option: -# inject_into: path -# pagination_strategy: -# type: "CursorPagination" -# cursor_value: "{{ response._metadata.next }}" -# requester: -# path: "/v3/marketing/lists" -# authenticator: -# type: "BearerAuthenticator" -# api_token: "{{ config.apikey }}" -# request_parameters: -# page_size: 10 -# record_selector: -# extractor: -# field_pointer: ["result"] -# streams: -# - type: DeclarativeStream -# $options: -# name: "lists" -# primary_key: id -# url_base: "https://api.sendgrid.com" -# schema_loader: "*ref(definitions.schema_loader)" -# retriever: "*ref(definitions.retriever)" -# check: -# type: CheckStream -# stream_names: ["lists"] -# not_a_valid_field: "error" -# """ -# temporary_file = TestFileContent(content) -# with self.assertRaises(InvalidConnectorDefinitionException): -# YamlDeclarativeSource(temporary_file.filename) -# -# def test_source_missing_checker_fails_validation(self): -# content = """ -# version: "version" -# definitions: -# schema_loader: -# name: "{{ options.stream_name }}" -# file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" -# retriever: -# paginator: -# type: "DefaultPaginator" -# page_size: 10 -# page_size_option: -# inject_into: request_parameter -# field_name: page_size -# page_token_option: -# inject_into: path -# pagination_strategy: -# type: "CursorPagination" -# cursor_value: "{{ response._metadata.next }}" -# requester: -# path: "/v3/marketing/lists" -# authenticator: -# type: "BearerAuthenticator" -# api_token: "{{ config.apikey }}" -# request_parameters: -# page_size: 10 -# record_selector: -# extractor: -# field_pointer: ["result"] -# streams: -# - type: DeclarativeStream -# $options: -# name: "lists" -# primary_key: id -# url_base: "https://api.sendgrid.com" -# schema_loader: "*ref(definitions.schema_loader)" -# retriever: "*ref(definitions.retriever)" -# check: -# type: CheckStream -# """ -# temporary_file = TestFileContent(content) -# with pytest.raises(ValidationError): -# YamlDeclarativeSource(temporary_file.filename) -# -# def test_source_with_missing_streams_fails(self): -# content = """ -# version: "version" -# definitions: -# check: -# type: CheckStream -# stream_names: ["lists"] -# """ -# temporary_file = TestFileContent(content) -# with pytest.raises(ValidationError): -# YamlDeclarativeSource(temporary_file.filename) -# -# def test_source_with_missing_version_fails(self): -# content = """ -# definitions: -# schema_loader: -# name: "{{ options.stream_name }}" -# file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" -# retriever: -# paginator: -# type: "DefaultPaginator" -# page_size: 10 -# page_size_option: -# inject_into: request_parameter -# field_name: page_size -# page_token_option: -# inject_into: path -# pagination_strategy: -# type: "CursorPagination" -# cursor_value: "{{ response._metadata.next }}" -# requester: -# path: "/v3/marketing/lists" -# authenticator: -# type: "BearerAuthenticator" -# api_token: "{{ config.apikey }}" -# request_parameters: -# page_size: 10 -# record_selector: -# extractor: -# field_pointer: ["result"] -# streams: -# - type: DeclarativeStream -# $options: -# name: "lists" -# primary_key: id -# url_base: "https://api.sendgrid.com" -# schema_loader: "*ref(definitions.schema_loader)" -# retriever: "*ref(definitions.retriever)" -# check: -# type: CheckStream -# stream_names: ["lists"] -# """ -# temporary_file = TestFileContent(content) -# with pytest.raises(ValidationError): -# YamlDeclarativeSource(temporary_file.filename) -# -# def test_source_with_invalid_stream_config_fails_validation(self): -# content = """ -# version: "version" -# definitions: -# schema_loader: -# name: "{{ options.stream_name }}" -# file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" -# streams: -# - type: DeclarativeStream -# $options: -# name: "lists" -# primary_key: id -# url_base: "https://api.sendgrid.com" -# schema_loader: "*ref(definitions.schema_loader)" -# check: -# type: CheckStream -# stream_names: ["lists"] -# """ -# temporary_file = TestFileContent(content) -# with pytest.raises(ValidationError): -# YamlDeclarativeSource(temporary_file.filename) -# -# -# class TestFileContent: -# def __init__(self, content): -# self.file = tempfile.NamedTemporaryFile(mode="w", delete=False) -# -# with self.file as f: -# f.write(content) -# -# @property -# def filename(self): -# return self.file.name -# -# def __enter__(self): -# return self -# -# def __exit__(self, type, value, traceback): -# os.unlink(self.filename) +logger = logging.getLogger("airbyte") + +EXTERNAL_CONNECTION_SPECIFICATION = { + "type": "object", + "required": ["api_token"], + "additionalProperties": False, + "properties": {"api_token": {"type": "string"}}, +} + + +class MockYamlDeclarativeSource(YamlDeclarativeSource): + """ + Mock test class that is needed to monkey patch how we read from various files that make up a declarative source because of how our + tests write configuration files during testing. It is also used to properly namespace where files get written in specific + cases like when we temporarily write files like spec.yaml to the package unit_tests, which is the directory where it will + be read in during the tests. + """ + + def _read_and_parse_yaml_file(self, path_to_yaml_file): + """ + We override the default behavior because we use tempfile to write the yaml manifest to a temporary directory which is + not mounted during runtime which prevents pkgutil.get_data() from being able to find the yaml file needed to generate + # the declarative source. For tests we use open() which supports using an absolute path. + """ + with open(path_to_yaml_file, "r") as f: + config_content = f.read() + parsed_config = YamlParser().parse(config_content) + return parsed_config + + +class TestYamlDeclarativeSource: + @pytest.fixture + def use_external_yaml_spec(self): + # Our way of resolving the absolute path to root of the airbyte-cdk unit test directory where spec.yaml files should + # be written to (i.e. ~/airbyte/airbyte-cdk/python/unit-tests) because that is where they are read from during testing. + module = sys.modules[__name__] + module_path = os.path.abspath(module.__file__) + test_path = os.path.dirname(module_path) + spec_root = test_path.split("/sources/declarative")[0] + + spec = {"documentationUrl": "https://airbyte.com/#yaml-from-external", "connectionSpecification": EXTERNAL_CONNECTION_SPECIFICATION} + + yaml_path = os.path.join(spec_root, "spec.yaml") + with open(yaml_path, "w") as f: + f.write(yaml.dump(spec)) + yield + os.remove(yaml_path) + + def test_source_is_created_if_toplevel_fields_are_known(self): + content = """ + version: "version" + definitions: + schema_loader: + name: "{{ options.stream_name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" + retriever: + paginator: + type: "DefaultPaginator" + page_size: 10 + page_size_option: + inject_into: request_parameter + field_name: page_size + page_token_option: + inject_into: path + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response._metadata.next }}" + requester: + path: "/v3/marketing/lists" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config.apikey }}" + request_parameters: + page_size: 10 + record_selector: + extractor: + field_pointer: ["result"] + streams: + - type: DeclarativeStream + $options: + name: "lists" + primary_key: id + url_base: "https://api.sendgrid.com" + schema_loader: "*ref(definitions.schema_loader)" + retriever: "*ref(definitions.retriever)" + check: + type: CheckStream + stream_names: ["lists"] + """ + temporary_file = TestFileContent(content) + MockYamlDeclarativeSource(temporary_file.filename) + + def test_source_with_spec_in_yaml(self): + content = """ + version: "version" + definitions: + schema_loader: + name: "{{ options.stream_name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" + retriever: + paginator: + type: "DefaultPaginator" + page_size: 10 + page_size_option: + inject_into: request_parameter + field_name: page_size + page_token_option: + inject_into: path + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response._metadata.next }}" + requester: + path: "/v3/marketing/lists" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config.apikey }}" + request_parameters: + page_size: 10 + record_selector: + extractor: + field_pointer: ["result"] + streams: + - type: DeclarativeStream + $options: + name: "lists" + primary_key: id + url_base: "https://api.sendgrid.com" + schema_loader: "*ref(definitions.schema_loader)" + retriever: "*ref(definitions.retriever)" + check: + type: CheckStream + stream_names: ["lists"] + spec: + type: Spec + documentation_url: https://airbyte.com/#yaml-from-manifest + connection_specification: + title: Test Spec + type: object + required: + - api_key + additionalProperties: false + properties: + api_key: + type: string + airbyte_secret: true + title: API Key + description: Test API Key + order: 0 + """ + temporary_file = TestFileContent(content) + source = MockYamlDeclarativeSource(temporary_file.filename) + + connector_specification = source.spec(logger) + assert connector_specification is not None + assert connector_specification.documentationUrl == "https://airbyte.com/#yaml-from-manifest" + assert connector_specification.connectionSpecification["title"] == "Test Spec" + assert connector_specification.connectionSpecification["required"][0] == "api_key" + assert connector_specification.connectionSpecification["additionalProperties"] is False + assert connector_specification.connectionSpecification["properties"]["api_key"] == { + "type": "string", + "airbyte_secret": True, + "title": "API Key", + "description": "Test API Key", + "order": 0, + } + + def test_source_with_external_spec(self, use_external_yaml_spec): + content = """ + version: "version" + definitions: + schema_loader: + name: "{{ options.stream_name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" + retriever: + paginator: + type: "DefaultPaginator" + page_size: 10 + page_size_option: + inject_into: request_parameter + field_name: page_size + page_token_option: + inject_into: path + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response._metadata.next }}" + requester: + path: "/v3/marketing/lists" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config.apikey }}" + request_parameters: + page_size: 10 + record_selector: + extractor: + field_pointer: ["result"] + streams: + - type: DeclarativeStream + $options: + name: "lists" + primary_key: id + url_base: "https://api.sendgrid.com" + schema_loader: "*ref(definitions.schema_loader)" + retriever: "*ref(definitions.retriever)" + check: + type: CheckStream + stream_names: ["lists"] + """ + temporary_file = TestFileContent(content) + source = MockYamlDeclarativeSource(temporary_file.filename) + + connector_specification = source.spec(logger) + + assert connector_specification.documentationUrl == "https://airbyte.com/#yaml-from-external" + assert connector_specification.connectionSpecification == EXTERNAL_CONNECTION_SPECIFICATION + + def test_source_is_not_created_if_toplevel_fields_are_unknown(self): + content = """ + version: "version" + definitions: + schema_loader: + name: "{{ options.stream_name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" + retriever: + paginator: + type: "DefaultPaginator" + page_size: 10 + page_size_option: + inject_into: request_parameter + field_name: page_size + page_token_option: + inject_into: path + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response._metadata.next }}" + requester: + path: "/v3/marketing/lists" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config.apikey }}" + request_parameters: + page_size: 10 + record_selector: + extractor: + field_pointer: ["result"] + streams: + - type: DeclarativeStream + $options: + name: "lists" + primary_key: id + url_base: "https://api.sendgrid.com" + schema_loader: "*ref(definitions.schema_loader)" + retriever: "*ref(definitions.retriever)" + check: + type: CheckStream + stream_names: ["lists"] + not_a_valid_field: "error" + """ + temporary_file = TestFileContent(content) + with pytest.raises(InvalidConnectorDefinitionException): + MockYamlDeclarativeSource(temporary_file.filename) + + def test_source_missing_checker_fails_validation(self): + content = """ + version: "version" + definitions: + schema_loader: + name: "{{ options.stream_name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" + retriever: + paginator: + type: "DefaultPaginator" + page_size: 10 + page_size_option: + inject_into: request_parameter + field_name: page_size + page_token_option: + inject_into: path + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response._metadata.next }}" + requester: + path: "/v3/marketing/lists" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config.apikey }}" + request_parameters: + page_size: 10 + record_selector: + extractor: + field_pointer: ["result"] + streams: + - type: DeclarativeStream + $options: + name: "lists" + primary_key: id + url_base: "https://api.sendgrid.com" + schema_loader: "*ref(definitions.schema_loader)" + retriever: "*ref(definitions.retriever)" + check: + type: CheckStream + """ + temporary_file = TestFileContent(content) + with pytest.raises(ValidationError): + MockYamlDeclarativeSource(temporary_file.filename) + + def test_source_with_missing_streams_fails(self): + content = """ + version: "version" + definitions: + check: + type: CheckStream + stream_names: ["lists"] + """ + temporary_file = TestFileContent(content) + with pytest.raises(ValidationError): + MockYamlDeclarativeSource(temporary_file.filename) + + def test_source_with_missing_version_fails(self): + content = """ + definitions: + schema_loader: + name: "{{ options.stream_name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" + retriever: + paginator: + type: "DefaultPaginator" + page_size: 10 + page_size_option: + inject_into: request_parameter + field_name: page_size + page_token_option: + inject_into: path + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response._metadata.next }}" + requester: + path: "/v3/marketing/lists" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config.apikey }}" + request_parameters: + page_size: 10 + record_selector: + extractor: + field_pointer: ["result"] + streams: + - type: DeclarativeStream + $options: + name: "lists" + primary_key: id + url_base: "https://api.sendgrid.com" + schema_loader: "*ref(definitions.schema_loader)" + retriever: "*ref(definitions.retriever)" + check: + type: CheckStream + stream_names: ["lists"] + """ + temporary_file = TestFileContent(content) + with pytest.raises(ValidationError): + MockYamlDeclarativeSource(temporary_file.filename) + + def test_source_with_invalid_stream_config_fails_validation(self): + content = """ + version: "version" + definitions: + schema_loader: + name: "{{ options.stream_name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" + streams: + - type: DeclarativeStream + $options: + name: "lists" + primary_key: id + url_base: "https://api.sendgrid.com" + schema_loader: "*ref(definitions.schema_loader)" + check: + type: CheckStream + stream_names: ["lists"] + """ + temporary_file = TestFileContent(content) + with pytest.raises(ValidationError): + MockYamlDeclarativeSource(temporary_file.filename) + + def test_source_with_no_external_spec_and_no_in_yaml_spec_fails(self): + content = """ + version: "version" + definitions: + schema_loader: + name: "{{ options.stream_name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" + retriever: + paginator: + type: "DefaultPaginator" + page_size: 10 + page_size_option: + inject_into: request_parameter + field_name: page_size + page_token_option: + inject_into: path + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response._metadata.next }}" + requester: + path: "/v3/marketing/lists" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config.apikey }}" + request_parameters: + page_size: 10 + record_selector: + extractor: + field_pointer: ["result"] + streams: + - type: DeclarativeStream + $options: + name: "lists" + primary_key: id + url_base: "https://api.sendgrid.com" + schema_loader: "*ref(definitions.schema_loader)" + retriever: "*ref(definitions.retriever)" + check: + type: CheckStream + stream_names: ["lists"] + """ + temporary_file = TestFileContent(content) + source = MockYamlDeclarativeSource(temporary_file.filename) + + # We expect to fail here because we have not created a temporary spec.yaml file + with pytest.raises(FileNotFoundError): + source.spec(logger) + + +class TestFileContent: + def __init__(self, content): + self.file = tempfile.NamedTemporaryFile(mode="w", delete=False) + + with self.file as f: + f.write(content) + + @property + def filename(self): + return self.file.name + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + os.unlink(self.filename) def test_generate_schema(): diff --git a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/spec.yaml.hbs b/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/spec.yaml.hbs deleted file mode 100644 index 1c65d8c8502bc..0000000000000 --- a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/spec.yaml.hbs +++ /dev/null @@ -1,13 +0,0 @@ -documentationUrl: https://docsurl.com -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: {{capitalCase name}} Spec - type: object - required: - - api_key - additionalProperties: true - properties: - # 'TODO: This schema defines the configuration required for the source. This usually involves metadata such as database and/or authentication information.': - api_key: - type: string - description: API Key diff --git a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs b/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs index 2f75e22bbbd1d..c9a620037322e 100644 --- a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs @@ -33,3 +33,17 @@ streams: check: stream_names: - "customers" + +spec: + documentation_url: https://docsurl.com + connection_specification: + title: {{capitalCase name}} Spec + type: object + required: + - api_key + additionalProperties: true + properties: + # 'TODO: This schema defines the configuration required for the source. This usually involves metadata such as database and/or authentication information.': + api_key: + type: string + description: API Key