From e774e0c9f3d96343a124fe94f19ab6344b3fdf07 Mon Sep 17 00:00:00 2001 From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com> Date: Tue, 1 Aug 2023 18:53:36 +0300 Subject: [PATCH 1/4] feat: Add override_columns to openapi transformer --- cloudquery/sdk/transformers/openapi.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/cloudquery/sdk/transformers/openapi.py b/cloudquery/sdk/transformers/openapi.py index 23af197..aa980dc 100644 --- a/cloudquery/sdk/transformers/openapi.py +++ b/cloudquery/sdk/transformers/openapi.py @@ -23,12 +23,21 @@ def oapi_type_to_arrow_type(field) -> pa.DataType: else: return pa.string() +def get_column_by_name(columns: List[Column], name: str) -> Column: + for column in columns: + if column.name == name: + return column + return None -def oapi_definition_to_columns(definition: Dict) -> List[Column]: +def oapi_definition_to_columns(definition: Dict, override_columns=[]) -> List[Column]: columns = [] for key, value in definition["properties"].items(): column_type = oapi_type_to_arrow_type(value) - columns.append( - Column(name=key, type=column_type, description=value.get("description")) - ) + column = Column(name=key, type=column_type, description=value.get("description")) + override_column = get_column_by_name(override_columns, key) + if override_column is not None: + column.type = override_column.type + column.primary_key = override_column.primary_key + column.unique = override_column.unique + columns.append(column) return columns From 4c34d950350f186e760f3a07b7975861ea4fcfde Mon Sep 17 00:00:00 2001 From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com> Date: Tue, 1 Aug 2023 18:54:33 +0300 Subject: [PATCH 2/4] fmt --- cloudquery/sdk/transformers/openapi.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cloudquery/sdk/transformers/openapi.py b/cloudquery/sdk/transformers/openapi.py index aa980dc..f12710c 100644 --- a/cloudquery/sdk/transformers/openapi.py +++ b/cloudquery/sdk/transformers/openapi.py @@ -23,17 +23,21 @@ def oapi_type_to_arrow_type(field) -> pa.DataType: else: return pa.string() + def get_column_by_name(columns: List[Column], name: str) -> Column: for column in columns: if column.name == name: return column return None + def oapi_definition_to_columns(definition: Dict, override_columns=[]) -> List[Column]: columns = [] for key, value in definition["properties"].items(): column_type = oapi_type_to_arrow_type(value) - column = Column(name=key, type=column_type, description=value.get("description")) + column = Column( + name=key, type=column_type, description=value.get("description") + ) override_column = get_column_by_name(override_columns, key) if override_column is not None: column.type = override_column.type From f23393bbdeac780649f65df0bf14509c8e537323 Mon Sep 17 00:00:00 2001 From: Yevgeny Pats Date: Tue, 1 Aug 2023 19:17:43 +0300 Subject: [PATCH 3/4] Update cloudquery/sdk/transformers/openapi.py Co-authored-by: Herman Schaaf --- cloudquery/sdk/transformers/openapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudquery/sdk/transformers/openapi.py b/cloudquery/sdk/transformers/openapi.py index f12710c..e2f6eff 100644 --- a/cloudquery/sdk/transformers/openapi.py +++ b/cloudquery/sdk/transformers/openapi.py @@ -24,7 +24,7 @@ def oapi_type_to_arrow_type(field) -> pa.DataType: return pa.string() -def get_column_by_name(columns: List[Column], name: str) -> Column: +def get_column_by_name(columns: List[Column], name: str) -> Optional[Column]: for column in columns: if column.name == name: return column From c3f59fd3fdc098285b16fea493d9aa0e383911de Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Tue, 1 Aug 2023 17:44:35 +0100 Subject: [PATCH 4/4] missing import --- cloudquery/sdk/transformers/openapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudquery/sdk/transformers/openapi.py b/cloudquery/sdk/transformers/openapi.py index e2f6eff..300acce 100644 --- a/cloudquery/sdk/transformers/openapi.py +++ b/cloudquery/sdk/transformers/openapi.py @@ -1,4 +1,4 @@ -from typing import Dict, List +from typing import Dict, List, Optional import pyarrow as pa from cloudquery.sdk.types import JSONType from cloudquery.sdk.schema import Column