Source Klaviyo: update schema validation in SAT (#6952)

* Disable schema validation for configured_catalog * Upd pydantic-generated schema * Upd comment * Bump version, upd changelog
airbytehq · Oct 21, 2021 · dfdbda8 · dfdbda8
1 parent d5e80d0
commit dfdbda8
Show file tree

Hide file tree

Showing 6 changed files with 23 additions and 7 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/95e8cffd-b8c4-4039-968e-d32fb4a69bde.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/95e8cffd-b8c4-4039-968e-d32fb4a69bde.json
@@ -2,6 +2,6 @@
   "sourceDefinitionId": "95e8cffd-b8c4-4039-968e-d32fb4a69bde",
   "name": "Klaviyo",
   "dockerRepository": "airbyte/source-klaviyo",
-  "dockerImageTag": "0.1.1",
+  "dockerImageTag": "0.1.2",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/klaviyo"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -168,7 +168,7 @@
 - sourceDefinitionId: 95e8cffd-b8c4-4039-968e-d32fb4a69bde
   name: Klaviyo
   dockerRepository: airbyte/source-klaviyo
-  dockerImageTag: 0.1.1
+  dockerImageTag: 0.1.2
   documentationUrl: https://docs.airbyte.io/integrations/sources/klaviyo
   sourceType: api
 - sourceDefinitionId: 9da77001-af33-4bcd-be46-6252bf9342b9

diff --git a/airbyte-integrations/connectors/source-klaviyo/Dockerfile b/airbyte-integrations/connectors/source-klaviyo/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.1
+LABEL io.airbyte.version=0.1.2
 LABEL io.airbyte.name=airbyte/source-klaviyo
diff --git a/airbyte-integrations/connectors/source-klaviyo/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-klaviyo/integration_tests/invalid_config.json
@@ -1,3 +1,4 @@
 {
-  "todo-wrong-field": "this should be an incomplete config file, used in standard tests"
+    "api_key": "api-key",
+    "start_date": "2021-01-01T00:00:00Z"
 }
diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas.py
@@ -15,14 +15,24 @@ class Config:
 
         @staticmethod
         def schema_extra(schema: MutableMapping[str, Any], model) -> None:
-            # Pydantic adds title to every attribute, this is too much, so we manually drop them
-            for prop in schema.get("properties", {}).values():
+            # Pydantic adds a title to each attribute, that is not needed, so we manually drop them.
+            # Also pydantic does not add a "null" option to a field marked as optional,
+            # so we add this functionality manually. Same for "$ref"
+            schema.pop("title", None)
+            for name, prop in schema.get("properties", {}).items():
                 prop.pop("title", None)
+                allow_none = model.__fields__[name].allow_none
+                if allow_none:
+                    if "type" in prop:
+                        prop["type"] = ["null", prop["type"]]
+                    elif "$ref" in prop:
+                        ref = prop.pop("$ref")
+                        prop["oneOf"] = [{"type": "null"}, {"$ref": ref}]
 
     object: str
 
 
-class PersonList(BaseModel):
+class PersonList(BaseSchemaModel):
     id: str
     name: str
     created: datetime

diff --git a/docs/integrations/sources/klaviyo.md b/docs/integrations/sources/klaviyo.md
@@ -46,3 +46,8 @@ The Klaviyo connector should not run into Klaviyo API limitations under normal u
 
 Please follow these [steps](https://help.klaviyo.com/hc/en-us/articles/115005062267-How-to-Manage-Your-Account-s-API-Keys#your-private-api-keys3) to obtain Private API Key for your account.
 
+## CHANGELOG
+
+| Version | Date       | Pull Request | Subject |
+| :------ | :--------  | :-----       | :------ |
+| `0.1.2` | 2021-10-19 | [6952](https://github.com/airbytehq/airbyte/pull/6952) | Update schema validation in SAT |