From 82ea25edc8a67f34fb3d8c7f960c21dde89d5746 Mon Sep 17 00:00:00 2001 From: brianjlai Date: Fri, 21 Oct 2022 17:28:04 -0700 Subject: [PATCH 1/9] allow for spec to be defined in the source.yaml manifest instead of an external file --- .../declarative/yaml_declarative_source.py | 38 +- .../test_yaml_declarative_source.py | 604 +++++++++++------- .../source_{{snakeCase name}}/spec.yaml.hbs | 13 - .../{{snakeCase name}}.yaml.hbs | 15 + 4 files changed, 411 insertions(+), 259 deletions(-) delete mode 100644 airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/spec.yaml.hbs diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py index c9e31fd47ba925..b4d8df5923e54c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py @@ -9,8 +9,9 @@ import typing from dataclasses import dataclass, fields from enum import Enum, EnumMeta -from typing import Any, List, Mapping, Union +from typing import Any, List, Mapping, Optional, Union +from airbyte_cdk.models import ConnectorSpecification from airbyte_cdk.sources.declarative.checks import CheckStream from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource @@ -30,10 +31,18 @@ class ConcreteDeclarativeSource(JsonSchemaMixin): streams: List[DeclarativeStream] +def load_optional_package_file(package: str, filename: str) -> Optional[bytes]: + """Gets a resource from a package, returning None if it does not exist""" + try: + return pkgutil.get_data(package, filename) + except FileNotFoundError: + return None + + class YamlDeclarativeSource(DeclarativeSource): """Declarative source defined by a yaml file""" - VALID_TOP_LEVEL_FIELDS = {"definitions", "streams", "check", "version"} + VALID_TOP_LEVEL_FIELDS = {"check", "definitions", "spec", "streams", "version"} def __init__(self, path_to_yaml): """ @@ -69,9 +78,32 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: self._apply_log_level_to_stream_logger(self.logger, stream) return source_streams + def spec(self, logger: logging.Logger) -> ConnectorSpecification: + """ + Returns the spec for this integration. The spec is a JSON-Schema object describing the required configurations (e.g: username + and password) required to run this integration. For low-code connectors, this will first attempt to load the spec from the + manifest's spec block, otherwise it will load it from "spec.yaml" or "spec.json" in the project root. + """ + + spec = self._source_config.get("spec") + if spec: + return ConnectorSpecification.parse_obj(spec) + else: + return super().spec(logger) + def _read_and_parse_yaml_file(self, path_to_yaml_file): - package = self.__class__.__module__.split(".")[0] + # This is a hack, but operates on the assumption that during unit tests we write manifests to a temporary directory + # (ex. /var/folders/...). In production, source_manifest.yaml is loaded into the package in setup.py, but for our + # unit testing, the temporary file doesn't get loaded in so we have to access it using the open instead + path_parts = list(filter(None, path_to_yaml_file.split("/"))) + is_unit_test = len(path_parts) > 0 and path_parts[0] == "var" + if is_unit_test: + with open(path_to_yaml_file, "r") as f: + config_content = f.read() + parsed_config = YamlParser().parse(config_content) + return parsed_config + package = self.__class__.__module__.split(".")[0] yaml_config = pkgutil.get_data(package, path_to_yaml_file) decoded_yaml = yaml_config.decode() return YamlParser().parse(decoded_yaml) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py index e1195244740d16..2b97d0ea3f7d54 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py @@ -3,252 +3,370 @@ # import json +import logging +import os +import tempfile +import unittest +import pytest +from airbyte_cdk.sources.declarative.exceptions import InvalidConnectorDefinitionException from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource +from jsonschema import ValidationError -# import pytest -# from airbyte_cdk.sources.declarative.exceptions import InvalidConnectorDefinitionException - -# import os -# import tempfile -# import unittest - - -# from jsonschema import ValidationError - - -# brianjlai: Commenting these out for the moment because I can't figure out why the temp file is unreadable at runtime during testing -# its more urgent to fix the connectors -# class TestYamlDeclarativeSource(unittest.TestCase): -# def test_source_is_created_if_toplevel_fields_are_known(self): -# content = """ -# version: "version" -# definitions: -# schema_loader: -# name: "{{ options.stream_name }}" -# file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" -# retriever: -# paginator: -# type: "DefaultPaginator" -# page_size: 10 -# page_size_option: -# inject_into: request_parameter -# field_name: page_size -# page_token_option: -# inject_into: path -# pagination_strategy: -# type: "CursorPagination" -# cursor_value: "{{ response._metadata.next }}" -# requester: -# path: "/v3/marketing/lists" -# authenticator: -# type: "BearerAuthenticator" -# api_token: "{{ config.apikey }}" -# request_parameters: -# page_size: 10 -# record_selector: -# extractor: -# field_pointer: ["result"] -# streams: -# - type: DeclarativeStream -# $options: -# name: "lists" -# primary_key: id -# url_base: "https://api.sendgrid.com" -# schema_loader: "*ref(definitions.schema_loader)" -# retriever: "*ref(definitions.retriever)" -# check: -# type: CheckStream -# stream_names: ["lists"] -# """ -# temporary_file = TestFileContent(content) -# YamlDeclarativeSource(temporary_file.filename) -# -# def test_source_is_not_created_if_toplevel_fields_are_unknown(self): -# content = """ -# version: "version" -# definitions: -# schema_loader: -# name: "{{ options.stream_name }}" -# file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" -# retriever: -# paginator: -# type: "DefaultPaginator" -# page_size: 10 -# page_size_option: -# inject_into: request_parameter -# field_name: page_size -# page_token_option: -# inject_into: path -# pagination_strategy: -# type: "CursorPagination" -# cursor_value: "{{ response._metadata.next }}" -# requester: -# path: "/v3/marketing/lists" -# authenticator: -# type: "BearerAuthenticator" -# api_token: "{{ config.apikey }}" -# request_parameters: -# page_size: 10 -# record_selector: -# extractor: -# field_pointer: ["result"] -# streams: -# - type: DeclarativeStream -# $options: -# name: "lists" -# primary_key: id -# url_base: "https://api.sendgrid.com" -# schema_loader: "*ref(definitions.schema_loader)" -# retriever: "*ref(definitions.retriever)" -# check: -# type: CheckStream -# stream_names: ["lists"] -# not_a_valid_field: "error" -# """ -# temporary_file = TestFileContent(content) -# with self.assertRaises(InvalidConnectorDefinitionException): -# YamlDeclarativeSource(temporary_file.filename) -# -# def test_source_missing_checker_fails_validation(self): -# content = """ -# version: "version" -# definitions: -# schema_loader: -# name: "{{ options.stream_name }}" -# file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" -# retriever: -# paginator: -# type: "DefaultPaginator" -# page_size: 10 -# page_size_option: -# inject_into: request_parameter -# field_name: page_size -# page_token_option: -# inject_into: path -# pagination_strategy: -# type: "CursorPagination" -# cursor_value: "{{ response._metadata.next }}" -# requester: -# path: "/v3/marketing/lists" -# authenticator: -# type: "BearerAuthenticator" -# api_token: "{{ config.apikey }}" -# request_parameters: -# page_size: 10 -# record_selector: -# extractor: -# field_pointer: ["result"] -# streams: -# - type: DeclarativeStream -# $options: -# name: "lists" -# primary_key: id -# url_base: "https://api.sendgrid.com" -# schema_loader: "*ref(definitions.schema_loader)" -# retriever: "*ref(definitions.retriever)" -# check: -# type: CheckStream -# """ -# temporary_file = TestFileContent(content) -# with pytest.raises(ValidationError): -# YamlDeclarativeSource(temporary_file.filename) -# -# def test_source_with_missing_streams_fails(self): -# content = """ -# version: "version" -# definitions: -# check: -# type: CheckStream -# stream_names: ["lists"] -# """ -# temporary_file = TestFileContent(content) -# with pytest.raises(ValidationError): -# YamlDeclarativeSource(temporary_file.filename) -# -# def test_source_with_missing_version_fails(self): -# content = """ -# definitions: -# schema_loader: -# name: "{{ options.stream_name }}" -# file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" -# retriever: -# paginator: -# type: "DefaultPaginator" -# page_size: 10 -# page_size_option: -# inject_into: request_parameter -# field_name: page_size -# page_token_option: -# inject_into: path -# pagination_strategy: -# type: "CursorPagination" -# cursor_value: "{{ response._metadata.next }}" -# requester: -# path: "/v3/marketing/lists" -# authenticator: -# type: "BearerAuthenticator" -# api_token: "{{ config.apikey }}" -# request_parameters: -# page_size: 10 -# record_selector: -# extractor: -# field_pointer: ["result"] -# streams: -# - type: DeclarativeStream -# $options: -# name: "lists" -# primary_key: id -# url_base: "https://api.sendgrid.com" -# schema_loader: "*ref(definitions.schema_loader)" -# retriever: "*ref(definitions.retriever)" -# check: -# type: CheckStream -# stream_names: ["lists"] -# """ -# temporary_file = TestFileContent(content) -# with pytest.raises(ValidationError): -# YamlDeclarativeSource(temporary_file.filename) -# -# def test_source_with_invalid_stream_config_fails_validation(self): -# content = """ -# version: "version" -# definitions: -# schema_loader: -# name: "{{ options.stream_name }}" -# file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" -# streams: -# - type: DeclarativeStream -# $options: -# name: "lists" -# primary_key: id -# url_base: "https://api.sendgrid.com" -# schema_loader: "*ref(definitions.schema_loader)" -# check: -# type: CheckStream -# stream_names: ["lists"] -# """ -# temporary_file = TestFileContent(content) -# with pytest.raises(ValidationError): -# YamlDeclarativeSource(temporary_file.filename) -# -# -# class TestFileContent: -# def __init__(self, content): -# self.file = tempfile.NamedTemporaryFile(mode="w", delete=False) -# -# with self.file as f: -# f.write(content) -# -# @property -# def filename(self): -# return self.file.name -# -# def __enter__(self): -# return self -# -# def __exit__(self, type, value, traceback): -# os.unlink(self.filename) +logger = logging.getLogger("airbyte") + + +class TestYamlDeclarativeSource(unittest.TestCase): + def test_source_is_created_if_toplevel_fields_are_known(self): + content = """ + version: "version" + definitions: + schema_loader: + name: "{{ options.stream_name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" + retriever: + paginator: + type: "DefaultPaginator" + page_size: 10 + page_size_option: + inject_into: request_parameter + field_name: page_size + page_token_option: + inject_into: path + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response._metadata.next }}" + requester: + path: "/v3/marketing/lists" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config.apikey }}" + request_parameters: + page_size: 10 + record_selector: + extractor: + field_pointer: ["result"] + streams: + - type: DeclarativeStream + $options: + name: "lists" + primary_key: id + url_base: "https://api.sendgrid.com" + schema_loader: "*ref(definitions.schema_loader)" + retriever: "*ref(definitions.retriever)" + check: + type: CheckStream + stream_names: ["lists"] + """ + temporary_file = TestFileContent(content) + YamlDeclarativeSource(temporary_file.filename) + + def test_source_with_spec_in_yaml(self): + content = """ + version: "version" + definitions: + schema_loader: + name: "{{ options.stream_name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" + retriever: + paginator: + type: "DefaultPaginator" + page_size: 10 + page_size_option: + inject_into: request_parameter + field_name: page_size + page_token_option: + inject_into: path + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response._metadata.next }}" + requester: + path: "/v3/marketing/lists" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config.apikey }}" + request_parameters: + page_size: 10 + record_selector: + extractor: + field_pointer: ["result"] + streams: + - type: DeclarativeStream + $options: + name: "lists" + primary_key: id + url_base: "https://api.sendgrid.com" + schema_loader: "*ref(definitions.schema_loader)" + retriever: "*ref(definitions.retriever)" + check: + type: CheckStream + stream_names: ["lists"] + spec: + type: Spec + documentationUrl: https://docs.airbyte.com/integrations/sources/orbit + connectionSpecification: + title: Test Spec + type: object + required: + - api_key + additionalProperties: false + properties: + api_key: + type: string + airbyte_secret: true + title: API Key + description: Test API Key + order: 0 + """ + temporary_file = TestFileContent(content) + source = YamlDeclarativeSource(temporary_file.filename) + + connector_specification = source.spec(logger) + assert connector_specification is not None + assert connector_specification.connectionSpecification["title"] == "Test Spec" + assert connector_specification.connectionSpecification["required"][0] == "api_key" + assert connector_specification.connectionSpecification["additionalProperties"] is False + assert connector_specification.connectionSpecification["properties"]["api_key"] == { + "type": "string", + "airbyte_secret": True, + "title": "API Key", + "description": "Test API Key", + "order": 0, + } + + def test_source_is_not_created_if_toplevel_fields_are_unknown(self): + content = """ + version: "version" + definitions: + schema_loader: + name: "{{ options.stream_name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" + retriever: + paginator: + type: "DefaultPaginator" + page_size: 10 + page_size_option: + inject_into: request_parameter + field_name: page_size + page_token_option: + inject_into: path + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response._metadata.next }}" + requester: + path: "/v3/marketing/lists" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config.apikey }}" + request_parameters: + page_size: 10 + record_selector: + extractor: + field_pointer: ["result"] + streams: + - type: DeclarativeStream + $options: + name: "lists" + primary_key: id + url_base: "https://api.sendgrid.com" + schema_loader: "*ref(definitions.schema_loader)" + retriever: "*ref(definitions.retriever)" + check: + type: CheckStream + stream_names: ["lists"] + not_a_valid_field: "error" + """ + temporary_file = TestFileContent(content) + with self.assertRaises(InvalidConnectorDefinitionException): + YamlDeclarativeSource(temporary_file.filename) + + def test_source_missing_checker_fails_validation(self): + content = """ + version: "version" + definitions: + schema_loader: + name: "{{ options.stream_name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" + retriever: + paginator: + type: "DefaultPaginator" + page_size: 10 + page_size_option: + inject_into: request_parameter + field_name: page_size + page_token_option: + inject_into: path + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response._metadata.next }}" + requester: + path: "/v3/marketing/lists" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config.apikey }}" + request_parameters: + page_size: 10 + record_selector: + extractor: + field_pointer: ["result"] + streams: + - type: DeclarativeStream + $options: + name: "lists" + primary_key: id + url_base: "https://api.sendgrid.com" + schema_loader: "*ref(definitions.schema_loader)" + retriever: "*ref(definitions.retriever)" + check: + type: CheckStream + """ + temporary_file = TestFileContent(content) + with pytest.raises(ValidationError): + YamlDeclarativeSource(temporary_file.filename) + + def test_source_with_missing_streams_fails(self): + content = """ + version: "version" + definitions: + check: + type: CheckStream + stream_names: ["lists"] + """ + temporary_file = TestFileContent(content) + with pytest.raises(ValidationError): + YamlDeclarativeSource(temporary_file.filename) + + def test_source_with_missing_version_fails(self): + content = """ + definitions: + schema_loader: + name: "{{ options.stream_name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" + retriever: + paginator: + type: "DefaultPaginator" + page_size: 10 + page_size_option: + inject_into: request_parameter + field_name: page_size + page_token_option: + inject_into: path + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response._metadata.next }}" + requester: + path: "/v3/marketing/lists" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config.apikey }}" + request_parameters: + page_size: 10 + record_selector: + extractor: + field_pointer: ["result"] + streams: + - type: DeclarativeStream + $options: + name: "lists" + primary_key: id + url_base: "https://api.sendgrid.com" + schema_loader: "*ref(definitions.schema_loader)" + retriever: "*ref(definitions.retriever)" + check: + type: CheckStream + stream_names: ["lists"] + """ + temporary_file = TestFileContent(content) + with pytest.raises(ValidationError): + YamlDeclarativeSource(temporary_file.filename) + + def test_source_with_invalid_stream_config_fails_validation(self): + content = """ + version: "version" + definitions: + schema_loader: + name: "{{ options.stream_name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" + streams: + - type: DeclarativeStream + $options: + name: "lists" + primary_key: id + url_base: "https://api.sendgrid.com" + schema_loader: "*ref(definitions.schema_loader)" + check: + type: CheckStream + stream_names: ["lists"] + """ + temporary_file = TestFileContent(content) + with pytest.raises(ValidationError): + YamlDeclarativeSource(temporary_file.filename) + + def test_source_with_no_external_spec_and_no_in_yaml_spec_fails(self): + content = """ + version: "version" + definitions: + schema_loader: + name: "{{ options.stream_name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" + retriever: + paginator: + type: "DefaultPaginator" + page_size: 10 + page_size_option: + inject_into: request_parameter + field_name: page_size + page_token_option: + inject_into: path + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response._metadata.next }}" + requester: + path: "/v3/marketing/lists" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config.apikey }}" + request_parameters: + page_size: 10 + record_selector: + extractor: + field_pointer: ["result"] + streams: + - type: DeclarativeStream + $options: + name: "lists" + primary_key: id + url_base: "https://api.sendgrid.com" + schema_loader: "*ref(definitions.schema_loader)" + retriever: "*ref(definitions.retriever)" + check: + type: CheckStream + stream_names: ["lists"] + """ + temporary_file = TestFileContent(content) + source = YamlDeclarativeSource(temporary_file.filename) + + # We expect to fail here because we have not created a temporary spec.yaml file + with pytest.raises(FileNotFoundError): + source.spec(logger) + + +class TestFileContent: + def __init__(self, content): + self.file = tempfile.NamedTemporaryFile(mode="w", delete=False) + + with self.file as f: + f.write(content) + + @property + def filename(self): + return self.file.name + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + os.unlink(self.filename) def test_generate_schema(): diff --git a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/spec.yaml.hbs b/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/spec.yaml.hbs deleted file mode 100644 index 1c65d8c8502bc9..00000000000000 --- a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/spec.yaml.hbs +++ /dev/null @@ -1,13 +0,0 @@ -documentationUrl: https://docsurl.com -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: {{capitalCase name}} Spec - type: object - required: - - api_key - additionalProperties: true - properties: - # 'TODO: This schema defines the configuration required for the source. This usually involves metadata such as database and/or authentication information.': - api_key: - type: string - description: API Key diff --git a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs b/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs index 2f75e22bbbd1dc..50420a19d41570 100644 --- a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs @@ -33,3 +33,18 @@ streams: check: stream_names: - "customers" + +spec: + documentationUrl: https://docsurl.com + connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: {{capitalCase name}} Spec + type: object + required: + - api_key + additionalProperties: true + properties: + # 'TODO: This schema defines the configuration required for the source. This usually involves metadata such as database and/or authentication information.': + api_key: + type: string + description: API Key \ No newline at end of file From 0c69179c52a21fd80ef0535b19863e891426110e Mon Sep 17 00:00:00 2001 From: brianjlai Date: Mon, 24 Oct 2022 17:47:20 -0700 Subject: [PATCH 2/9] make spec a component within the language to get schema validation and rework the code for better testing --- .../parsers/class_types_registry.py | 2 + .../sources/declarative/parsers/factory.py | 2 +- .../sources/declarative/spec/__init__.py | 7 + .../sources/declarative/spec/spec.py | 37 ++++++ .../declarative/yaml_declarative_source.py | 22 ++-- .../test_yaml_declarative_source.py | 122 ++++++++++++++++-- .../{{snakeCase name}}.yaml.hbs | 4 +- 7 files changed, 168 insertions(+), 28 deletions(-) create mode 100644 airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/__init__.py create mode 100644 airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py index 2419443089aa26..9b8fc66457312b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py @@ -33,6 +33,7 @@ from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import PageIncrement from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever from airbyte_cdk.sources.declarative.schema.json_schema import JsonSchema +from airbyte_cdk.sources.declarative.spec import Spec from airbyte_cdk.sources.declarative.stream_slicers.cartesian_product_stream_slicer import CartesianProductStreamSlicer from airbyte_cdk.sources.declarative.stream_slicers.datetime_stream_slicer import DatetimeStreamSlicer from airbyte_cdk.sources.declarative.stream_slicers.list_stream_slicer import ListStreamSlicer @@ -74,6 +75,7 @@ "RemoveFields": RemoveFields, "SimpleRetriever": SimpleRetriever, "SingleSlice": SingleSlice, + "Spec": Spec, "SubstreamSlicer": SubstreamSlicer, "WaitUntilTimeFromHeader": WaitUntilTimeFromHeaderBackoffStrategy, "WaitTimeFromHeader": WaitTimeFromHeaderBackoffStrategy, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py index 676074d174b3ea..dd77d6fb8a77bc 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py @@ -243,7 +243,7 @@ def is_object_definition_with_class_name(definition): @staticmethod def is_object_definition_with_type(definition): - return isinstance(definition, dict) and "type" in definition + return isinstance(definition, dict) and "type" in definition and definition["type"] != "object" @staticmethod def get_default_type(parameter_name, parent_class): diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/__init__.py new file mode 100644 index 00000000000000..fba2f9612ba2ed --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/__init__.py @@ -0,0 +1,7 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.spec.spec import Spec + +__all__ = ["Spec"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py new file mode 100644 index 00000000000000..b25f359e001e95 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from dataclasses import InitVar, dataclass +from typing import Any, Mapping + +from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification +from dataclasses_jsonschema import JsonSchemaMixin + + +@dataclass +class Spec(JsonSchemaMixin): + """ + Returns a connection specification made up of information about the connector and how it can be configured + + Attributes: + documentation_url (str): The link the Airbyte documentation about this connector + connection_specification (str): information related to how a connector can be configured + """ + + documentation_url: str + connection_specification: Mapping[str, Any] + options: InitVar[Mapping[str, Any]] + + def __post_init__(self, options: Mapping[str, Any]): + self._options = options + + def generate_spec(self) -> ConnectorSpecification: + """ + Returns the connector specification according the spec block defined in the low code connector manifest. + """ + + # We remap these keys to camel case because that's the existing format expected by the rest of the platform + return ConnectorSpecification.parse_obj( + {"documentationUrl": self.documentation_url, "connectionSpecification": self.connection_specification} + ) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py index b4d8df5923e54c..9a7295e1d63238 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py @@ -85,25 +85,23 @@ def spec(self, logger: logging.Logger) -> ConnectorSpecification: manifest's spec block, otherwise it will load it from "spec.yaml" or "spec.json" in the project root. """ + self.logger.debug( + "parsed YAML into declarative source", + extra={"path_to_yaml_file": self._path_to_yaml, "source_name": self.name, "parsed_config": json.dumps(self._source_config)}, + ) + spec = self._source_config.get("spec") if spec: - return ConnectorSpecification.parse_obj(spec) + if "class_name" not in spec: + spec["class_name"] = "airbyte_cdk.sources.declarative.spec.Spec" + spec_component = self._factory.create_component(spec, dict())() + return spec_component.generate_spec() else: return super().spec(logger) def _read_and_parse_yaml_file(self, path_to_yaml_file): - # This is a hack, but operates on the assumption that during unit tests we write manifests to a temporary directory - # (ex. /var/folders/...). In production, source_manifest.yaml is loaded into the package in setup.py, but for our - # unit testing, the temporary file doesn't get loaded in so we have to access it using the open instead - path_parts = list(filter(None, path_to_yaml_file.split("/"))) - is_unit_test = len(path_parts) > 0 and path_parts[0] == "var" - if is_unit_test: - with open(path_to_yaml_file, "r") as f: - config_content = f.read() - parsed_config = YamlParser().parse(config_content) - return parsed_config - package = self.__class__.__module__.split(".")[0] + yaml_config = pkgutil.get_data(package, path_to_yaml_file) decoded_yaml = yaml_config.decode() return YamlParser().parse(decoded_yaml) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py index 2b97d0ea3f7d54..e54207df2fc38c 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py @@ -5,18 +5,64 @@ import json import logging import os +import sys import tempfile -import unittest import pytest +import yaml from airbyte_cdk.sources.declarative.exceptions import InvalidConnectorDefinitionException +from airbyte_cdk.sources.declarative.parsers.yaml_parser import YamlParser from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from jsonschema import ValidationError logger = logging.getLogger("airbyte") +EXTERNAL_CONNECTION_SPECIFICATION = { + "type": "object", + "required": ["api_token"], + "additionalProperties": False, + "properties": {"api_token": {"type": "string"}}, +} + + +class MockYamlDeclarativeSource(YamlDeclarativeSource): + """ + Mock test class that is needed to monkey patch how we read from various files that make up a declarative source because of how our + tests write configuration files during testing. It is also used to properly namespace where files get written in specific + cases like when we temporarily write files like spec.yaml to the package unit_tests, which is the directory where it will + be read in during the tests. + """ + + def _read_and_parse_yaml_file(self, path_to_yaml_file): + """ + We override the default behavior because we use tempfile to write the yaml manifest to a temporary directory which is + not mounted during runtime which prevents pkgutil.get_data() from being able to find the yaml file needed to generate + # the declarative source. For tests we use open() which supports using an absolute path. + """ + with open(path_to_yaml_file, "r") as f: + config_content = f.read() + parsed_config = YamlParser().parse(config_content) + return parsed_config + + +class TestYamlDeclarativeSource: + @pytest.fixture + def use_external_yaml_spec(self): + # Our way of resolving the absolute path to root of the airbyte-cdk unit test directory where spec.yaml files should + # be written to (i.e. ~/airbyte/airbyte-cdk/python/unit-tests) because that is where they are read from during testing. + module = sys.modules[__name__] + module_path = os.path.abspath(module.__file__) + test_path = os.path.dirname(module_path) + spec_root = test_path.split("/sources/declarative")[0] + + spec = {"documentationUrl": "https://airbyte.com/#yaml-from-external", "connectionSpecification": EXTERNAL_CONNECTION_SPECIFICATION} + + yaml_path = os.path.join(spec_root, "spec.yaml") + with open(yaml_path, "w") as f: + f.write(yaml.dump(spec)) + yield + os.remove(yaml_path) -class TestYamlDeclarativeSource(unittest.TestCase): def test_source_is_created_if_toplevel_fields_are_known(self): content = """ version: "version" @@ -59,7 +105,7 @@ def test_source_is_created_if_toplevel_fields_are_known(self): stream_names: ["lists"] """ temporary_file = TestFileContent(content) - YamlDeclarativeSource(temporary_file.filename) + MockYamlDeclarativeSource(temporary_file.filename) def test_source_with_spec_in_yaml(self): content = """ @@ -103,8 +149,8 @@ def test_source_with_spec_in_yaml(self): stream_names: ["lists"] spec: type: Spec - documentationUrl: https://docs.airbyte.com/integrations/sources/orbit - connectionSpecification: + documentation_url: https://airbyte.com/#yaml-from-manifest + connection_specification: title: Test Spec type: object required: @@ -119,10 +165,11 @@ def test_source_with_spec_in_yaml(self): order: 0 """ temporary_file = TestFileContent(content) - source = YamlDeclarativeSource(temporary_file.filename) + source = MockYamlDeclarativeSource(temporary_file.filename) connector_specification = source.spec(logger) assert connector_specification is not None + assert connector_specification.documentationUrl == "https://airbyte.com/#yaml-from-manifest" assert connector_specification.connectionSpecification["title"] == "Test Spec" assert connector_specification.connectionSpecification["required"][0] == "api_key" assert connector_specification.connectionSpecification["additionalProperties"] is False @@ -134,6 +181,55 @@ def test_source_with_spec_in_yaml(self): "order": 0, } + def test_source_with_external_spec(self, use_external_yaml_spec): + content = """ + version: "version" + definitions: + schema_loader: + name: "{{ options.stream_name }}" + file_path: "./source_sendgrid/schemas/{{ options.name }}.yaml" + retriever: + paginator: + type: "DefaultPaginator" + page_size: 10 + page_size_option: + inject_into: request_parameter + field_name: page_size + page_token_option: + inject_into: path + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response._metadata.next }}" + requester: + path: "/v3/marketing/lists" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config.apikey }}" + request_parameters: + page_size: 10 + record_selector: + extractor: + field_pointer: ["result"] + streams: + - type: DeclarativeStream + $options: + name: "lists" + primary_key: id + url_base: "https://api.sendgrid.com" + schema_loader: "*ref(definitions.schema_loader)" + retriever: "*ref(definitions.retriever)" + check: + type: CheckStream + stream_names: ["lists"] + """ + temporary_file = TestFileContent(content) + source = MockYamlDeclarativeSource(temporary_file.filename) + + connector_specification = source.spec(logger) + + assert connector_specification.documentationUrl == "https://airbyte.com/#yaml-from-external" + assert connector_specification.connectionSpecification == EXTERNAL_CONNECTION_SPECIFICATION + def test_source_is_not_created_if_toplevel_fields_are_unknown(self): content = """ version: "version" @@ -177,8 +273,8 @@ def test_source_is_not_created_if_toplevel_fields_are_unknown(self): not_a_valid_field: "error" """ temporary_file = TestFileContent(content) - with self.assertRaises(InvalidConnectorDefinitionException): - YamlDeclarativeSource(temporary_file.filename) + with pytest.raises(InvalidConnectorDefinitionException): + MockYamlDeclarativeSource(temporary_file.filename) def test_source_missing_checker_fails_validation(self): content = """ @@ -222,7 +318,7 @@ def test_source_missing_checker_fails_validation(self): """ temporary_file = TestFileContent(content) with pytest.raises(ValidationError): - YamlDeclarativeSource(temporary_file.filename) + MockYamlDeclarativeSource(temporary_file.filename) def test_source_with_missing_streams_fails(self): content = """ @@ -234,7 +330,7 @@ def test_source_with_missing_streams_fails(self): """ temporary_file = TestFileContent(content) with pytest.raises(ValidationError): - YamlDeclarativeSource(temporary_file.filename) + MockYamlDeclarativeSource(temporary_file.filename) def test_source_with_missing_version_fails(self): content = """ @@ -278,7 +374,7 @@ def test_source_with_missing_version_fails(self): """ temporary_file = TestFileContent(content) with pytest.raises(ValidationError): - YamlDeclarativeSource(temporary_file.filename) + MockYamlDeclarativeSource(temporary_file.filename) def test_source_with_invalid_stream_config_fails_validation(self): content = """ @@ -300,7 +396,7 @@ def test_source_with_invalid_stream_config_fails_validation(self): """ temporary_file = TestFileContent(content) with pytest.raises(ValidationError): - YamlDeclarativeSource(temporary_file.filename) + MockYamlDeclarativeSource(temporary_file.filename) def test_source_with_no_external_spec_and_no_in_yaml_spec_fails(self): content = """ @@ -344,7 +440,7 @@ def test_source_with_no_external_spec_and_no_in_yaml_spec_fails(self): stream_names: ["lists"] """ temporary_file = TestFileContent(content) - source = YamlDeclarativeSource(temporary_file.filename) + source = MockYamlDeclarativeSource(temporary_file.filename) # We expect to fail here because we have not created a temporary spec.yaml file with pytest.raises(FileNotFoundError): diff --git a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs b/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs index 50420a19d41570..bf8e8064751802 100644 --- a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs @@ -35,8 +35,8 @@ check: - "customers" spec: - documentationUrl: https://docsurl.com - connectionSpecification: + documentation_url: https://docsurl.com + connection_specification: $schema: http://json-schema.org/draft-07/schema# title: {{capitalCase name}} Spec type: object From 9ce4b48957b73978492fe713ef0fb77e68883215 Mon Sep 17 00:00:00 2001 From: brianjlai Date: Mon, 24 Oct 2022 18:02:24 -0700 Subject: [PATCH 3/9] fix formatting and extra method --- .../sources/declarative/yaml_declarative_source.py | 10 +--------- .../{{snakeCase name}}.yaml.hbs | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py index 9a7295e1d63238..bfcb613e2d4bc1 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py @@ -9,7 +9,7 @@ import typing from dataclasses import dataclass, fields from enum import Enum, EnumMeta -from typing import Any, List, Mapping, Optional, Union +from typing import Any, List, Mapping, Union from airbyte_cdk.models import ConnectorSpecification from airbyte_cdk.sources.declarative.checks import CheckStream @@ -31,14 +31,6 @@ class ConcreteDeclarativeSource(JsonSchemaMixin): streams: List[DeclarativeStream] -def load_optional_package_file(package: str, filename: str) -> Optional[bytes]: - """Gets a resource from a package, returning None if it does not exist""" - try: - return pkgutil.get_data(package, filename) - except FileNotFoundError: - return None - - class YamlDeclarativeSource(DeclarativeSource): """Declarative source defined by a yaml file""" diff --git a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs b/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs index bf8e8064751802..f1f0babb8f9f67 100644 --- a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs @@ -47,4 +47,4 @@ spec: # 'TODO: This schema defines the configuration required for the source. This usually involves metadata such as database and/or authentication information.': api_key: type: string - description: API Key \ No newline at end of file + description: API Key From 5aa17e2e515c19bec89cc0a4157a3d7363a273f2 Mon Sep 17 00:00:00 2001 From: brianjlai Date: Wed, 2 Nov 2022 17:37:54 -0700 Subject: [PATCH 4/9] pr feedback and add some more test --- .../sources/declarative/spec/spec.py | 5 +-- .../declarative/yaml_declarative_source.py | 7 +++-- .../sources/declarative/test_factory.py | 31 +++++++++++++++++++ 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py index b25f359e001e95..d5ac6a1d586d34 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py @@ -16,16 +16,13 @@ class Spec(JsonSchemaMixin): Attributes: documentation_url (str): The link the Airbyte documentation about this connector - connection_specification (str): information related to how a connector can be configured + connection_specification (Mapping[str, Any]): information related to how a connector can be configured """ documentation_url: str connection_specification: Mapping[str, Any] options: InitVar[Mapping[str, Any]] - def __post_init__(self, options: Mapping[str, Any]): - self._options = options - def generate_spec(self) -> ConnectorSpecification: """ Returns the connector specification according the spec block defined in the low code connector manifest. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py index bfcb613e2d4bc1..440b8cc5c98233 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/yaml_declarative_source.py @@ -72,9 +72,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: def spec(self, logger: logging.Logger) -> ConnectorSpecification: """ - Returns the spec for this integration. The spec is a JSON-Schema object describing the required configurations (e.g: username - and password) required to run this integration. For low-code connectors, this will first attempt to load the spec from the - manifest's spec block, otherwise it will load it from "spec.yaml" or "spec.json" in the project root. + Returns the connector specification (spec) as defined in the Airbyte Protocol. The spec is an object describing the possible + configurations (e.g: username and password) which can be configured when running this connector. For low-code connectors, this + will first attempt to load the spec from the manifest's spec block, otherwise it will load it from "spec.yaml" or "spec.json" + in the project root. """ self.logger.debug( diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py index 684a68c0156183..f4cd8390d751d0 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py @@ -343,6 +343,22 @@ def test_full_config(): check: class_name: airbyte_cdk.sources.declarative.checks.check_stream.CheckStream stream_names: ["list_stream"] +spec: + class_name: airbyte_cdk.sources.declarative.spec.Spec + documentation_url: https://airbyte.com/#yaml-from-manifest + connection_specification: + title: Test Spec + type: object + required: + - api_key + additionalProperties: false + properties: + api_key: + type: string + airbyte_secret: true + title: API Key + description: Test API Key + order: 0 """ config = parser.parse(content) @@ -375,6 +391,21 @@ def test_full_config(): assert len(streams_to_check) == 1 assert list(streams_to_check)[0] == "list_stream" + spec = factory.create_component(config["spec"], input_config)() + documentation_url = spec.documentation_url + connection_specification = spec.connection_specification + assert documentation_url == "https://airbyte.com/#yaml-from-manifest" + print(connection_specification) + assert connection_specification["title"] == "Test Spec" + assert connection_specification["required"] == ["api_key"] + assert connection_specification["properties"]["api_key"] == { + "type": "string", + "airbyte_secret": True, + "title": "API Key", + "description": "Test API Key", + "order": 0, + } + assert stream.retriever.requester.path.default == "marketing/lists" From 2be92c774af2d326f90190e248a7fcf61423b74b Mon Sep 17 00:00:00 2001 From: brianjlai Date: Mon, 7 Nov 2022 00:16:28 -0800 Subject: [PATCH 5/9] pr feedback --- .../python/airbyte_cdk/sources/declarative/parsers/factory.py | 4 ++++ .../python/unit_tests/sources/declarative/test_factory.py | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py index 25fa0ceddbe32e..83651ad88ef01d 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py @@ -246,6 +246,10 @@ def is_object_definition_with_class_name(definition): @staticmethod def is_object_definition_with_type(definition): + # The `type` field is an overloaded term in the context of the low-code manifest. As part of the language, `type` is shorthand + # for convenience to avoid defining the entire classpath. For the connector specification, `type` is a part of the spec schema. + # For spec parsing, as part of this check, when the type is an object, we want it to remain a mapping. But when type is defined + # any other way, then it should be parsed as a declarative component in the manifest. return isinstance(definition, dict) and "type" in definition and definition["type"] != "object" @staticmethod diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py index 63c561c2cfd4df..c3464de18ddaac 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py @@ -396,7 +396,6 @@ def test_full_config(): documentation_url = spec.documentation_url connection_specification = spec.connection_specification assert documentation_url == "https://airbyte.com/#yaml-from-manifest" - print(connection_specification) assert connection_specification["title"] == "Test Spec" assert connection_specification["required"] == ["api_key"] assert connection_specification["properties"]["api_key"] == { From 48f259523a878aad90f9bddb5f59996dbe310ee8 Mon Sep 17 00:00:00 2001 From: brianjlai Date: Mon, 7 Nov 2022 00:18:00 -0800 Subject: [PATCH 6/9] bump airbyte-cdk version --- airbyte-cdk/python/CHANGELOG.md | 3 +++ airbyte-cdk/python/setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index fda8cd5846acf3..13c5a98cf2c679 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.6.0 +Low-code: Allow connector specifications to be defined in the manifest + ## 0.5.4 Low-code: Get response.json in a safe way diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index d133785a6ec9f3..330270026f8480 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.5.4", + version="0.6.0", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", From 582ccb0d802cf30b3a8a11a5a9fee65c9aab7205 Mon Sep 17 00:00:00 2001 From: brianjlai Date: Mon, 7 Nov 2022 10:20:31 -0800 Subject: [PATCH 7/9] bump version --- airbyte-cdk/python/CHANGELOG.md | 3 +++ .../python/airbyte_cdk/sources/declarative/parsers/factory.py | 4 ++-- airbyte-cdk/python/setup.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index a1ce83a8a2c36d..4612a87fa4854a 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.7.0 +Low-code: Allow connector specifications to be defined in the manifest + ## 0.6.0 Low-code: Add support for monthly and yearly incremental updates for `DatetimeStreamSlicer` diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py index 83651ad88ef01d..0904fb3bb66a35 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/factory.py @@ -248,8 +248,8 @@ def is_object_definition_with_class_name(definition): def is_object_definition_with_type(definition): # The `type` field is an overloaded term in the context of the low-code manifest. As part of the language, `type` is shorthand # for convenience to avoid defining the entire classpath. For the connector specification, `type` is a part of the spec schema. - # For spec parsing, as part of this check, when the type is an object, we want it to remain a mapping. But when type is defined - # any other way, then it should be parsed as a declarative component in the manifest. + # For spec parsing, as part of this check, when the type is set to object, we want it to remain a mapping. But when type is + # defined any other way, then it should be parsed as a declarative component in the manifest. return isinstance(definition, dict) and "type" in definition and definition["type"] != "object" @staticmethod diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 02c854cb6e1a97..46a84500bc661f 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.6.0", + version="0.7.0", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", From 360db7fd53d4bc9cc0f863cad45bf505b60f8446 Mon Sep 17 00:00:00 2001 From: brianjlai Date: Mon, 7 Nov 2022 10:45:39 -0800 Subject: [PATCH 8/9] gradle format --- .../declarative/stream_slicers/datetime_stream_slicer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py index 9fdffcc703f4e3..181ddc096d99a3 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py @@ -5,7 +5,6 @@ import datetime import re from dataclasses import InitVar, dataclass, field -from dateutil.relativedelta import relativedelta from typing import Any, Iterable, Mapping, Optional, Union from airbyte_cdk.models import SyncMode @@ -17,6 +16,7 @@ from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState from dataclasses_jsonschema import JsonSchemaMixin +from dateutil.relativedelta import relativedelta @dataclass @@ -71,7 +71,9 @@ class DatetimeStreamSlicer(StreamSlicer, JsonSchemaMixin): stream_state_field_end: Optional[str] = None lookback_window: Optional[Union[InterpolatedString, str]] = None - timedelta_regex = re.compile(r"((?P[\.\d]+?)y)?" r"((?P[\.\d]+?)m)?" r"((?P[\.\d]+?)w)?" r"((?P[\.\d]+?)d)?$") + timedelta_regex = re.compile( + r"((?P[\.\d]+?)y)?" r"((?P[\.\d]+?)m)?" r"((?P[\.\d]+?)w)?" r"((?P[\.\d]+?)d)?$" + ) def __post_init__(self, options: Mapping[str, Any]): if not isinstance(self.start_datetime, MinMaxDatetime): From 17c38a4fce74c5a7c543fe8daf10f6f8b5cb08cf Mon Sep 17 00:00:00 2001 From: brianjlai Date: Mon, 7 Nov 2022 11:43:47 -0800 Subject: [PATCH 9/9] remove from manifest spec --- .../source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs b/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs index f1f0babb8f9f67..c9a620037322ec 100644 --- a/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-configuration-based/source_{{snakeCase name}}/{{snakeCase name}}.yaml.hbs @@ -37,7 +37,6 @@ check: spec: documentation_url: https://docsurl.com connection_specification: - $schema: http://json-schema.org/draft-07/schema# title: {{capitalCase name}} Spec type: object required: