From 32cd4af57fb229041dc652a014c46890c5d17ac3 Mon Sep 17 00:00:00 2001 From: Dmytro Rezchykov Date: Mon, 8 Nov 2021 14:12:19 +0200 Subject: [PATCH] Resolve discover catalog ref fields --- airbyte-cdk/python/CHANGELOG.md | 3 +++ .../sources/utils/schema_helpers.py | 25 +++++++------------ airbyte-cdk/python/setup.py | 2 +- .../sources/utils/test_schema_helpers.py | 8 +++--- .../source-facebook-marketing/Dockerfile | 2 +- .../source-facebook-marketing/setup.py | 2 +- .../sources/facebook-marketing.md | 3 ++- 7 files changed, 20 insertions(+), 25 deletions(-) diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index f230eccf3fcc8..1429dc1869583 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.1.33 + Resolve $ref fields for discover json schema. + ## 0.1.32 - Added Sphinx docs `airbyte-cdk/python/reference_docs` module. - Added module documents at `airbyte-cdk/python/sphinx-docs.md`. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py index b29b83e2b8957..6362d33b7522e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py @@ -75,36 +75,29 @@ def __call__(self, uri: str) -> Dict[str, Any]: package = importlib.import_module(self.package_name) base = os.path.dirname(package.__file__) + "/" - def create_definitions(obj: dict, definitions: dict) -> Dict[str, Any]: + def resolve_ref_links(obj: Any) -> Dict[str, Any]: """ - Scan resolved schema and compose definitions section, also convert - jsonref.JsonRef object to JSON serializable dict. + Scan resolved schema and convert jsonref.JsonRef object to JSON + serializable dict. :param obj - jsonschema object with ref field resovled. - :definitions - object for storing generated definitions. :return JSON serializable object with references without external dependencies. """ if isinstance(obj, jsonref.JsonRef): - def_key = obj.__reference__["$ref"] - def_key = def_key.replace("#/definitions/", "").replace(".json", "_") - definition = create_definitions(obj.__subject__, definitions) + obj = resolve_ref_links(obj.__subject__) # Omit existance definitions for extenal resource since # we dont need it anymore. - definition.pop("definitions", None) - definitions[def_key] = definition - return {"$ref": "#/definitions/" + def_key} + obj.pop("definitions", None) + return obj elif isinstance(obj, dict): - return {k: create_definitions(v, definitions) for k, v in obj.items()} + return {k: resolve_ref_links(v) for k, v in obj.items()} elif isinstance(obj, list): - return [create_definitions(item, definitions) for item in obj] + return [resolve_ref_links(item) for item in obj] else: return obj resolved = jsonref.JsonRef.replace_refs(raw_schema, loader=JsonFileLoader(base, "schemas/shared"), base_uri=base) - definitions = {} - resolved = create_definitions(resolved, definitions) - if definitions: - resolved["definitions"] = definitions + resolved = resolve_ref_links(resolved) return resolved diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 72fa8b3dd5eb8..feba71a434bae 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.1.32", + version="0.1.33", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py b/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py index 267a1f865c369..a80c6c7e8e739 100644 --- a/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py +++ b/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py @@ -96,9 +96,8 @@ def test_shared_schemas_resolves(): "properties": { "str": {"type": "string"}, "int": {"type": "integer"}, - "obj": {"$ref": "#/definitions/shared_schema_"}, + "obj": {"type": ["null", "object"], "properties": {"k1": {"type": "string"}}}, }, - "definitions": {"shared_schema_": {"type": ["null", "object"], "properties": {"k1": {"type": "string"}}}}, } partial_schema = { @@ -123,10 +122,9 @@ def test_shared_schemas_resolves_nested(): "properties": { "str": {"type": "string"}, "int": {"type": "integer"}, - "one_of": {"oneOf": [{"type": "string"}, {"$ref": "#/definitions/shared_schema_type_one"}]}, - "obj": {"$ref": "#/definitions/shared_schema_type_one"}, + "one_of": {"oneOf": [{"type": "string"}, {"type": ["null", "object"], "properties": {"k1": {"type": "string"}}}]}, + "obj": {"type": ["null", "object"], "properties": {"k1": {"type": "string"}}}, }, - "definitions": {"shared_schema_type_one": {"type": ["null", "object"], "properties": {"k1": {"type": "string"}}}}, } partial_schema = { "type": ["null", "object"], diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index 2a2905c3f229e..11b20745b0fe5 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.22 +LABEL io.airbyte.version=0.2.23 LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/setup.py b/airbyte-integrations/connectors/source-facebook-marketing/setup.py index da171c44c163a..91242819cffb0 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/setup.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1.24", + "airbyte-cdk~=0.1.33", "cached_property~=1.5", "facebook_business~=12.0", "pendulum>=2,<3", diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index 1a6ef22e57872..0f1be8be1d238 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -96,7 +96,8 @@ As a summary, custom insights allows to replicate only some fields, resulting in | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.2.22 | 2021-11-05 | [4864](https://github.com/airbytehq/airbyte/pull/7605) | Add job retry logics to AdsInsights stream | +| 0.2.23 | 2021-11-08 | [7734](https://github.com/airbytehq/airbyte/pull/7734) | Resolve $ref field for discover schema | +| 0.2.22 | 2021-11-05 | [7605](https://github.com/airbytehq/airbyte/pull/7605) | Add job retry logics to AdsInsights stream | | 0.2.21 | 2021-10-05 | [4864](https://github.com/airbytehq/airbyte/pull/4864) | Update insights streams with custom entries for fields, breakdowns and action_breakdowns | | 0.2.20 | 2021-10-04 | [6719](https://github.com/airbytehq/airbyte/pull/6719) | Update version of facebook\_bussiness package to 12.0 | | 0.2.19 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification |