CDK: add option to source to skip config validation on read/discover (a…

…irbytehq#6978) * Add connector check_config_against_spec parameter * Bump CDK version. Add changelog record. * Always enable spec check on check cmd * Update airbyte-cdk/python/CHANGELOG.md Co-authored-by: Sherif A. Nada <snadalive@gmail.com> Co-authored-by: Sherif A. Nada <snadalive@gmail.com>
schlattk · Jan 4, 2022 · 5440125 · 5440125
1 parent 075171e
commit 5440125
Show file tree

Hide file tree

Showing 5 changed files with 12 additions and 3 deletions.
diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.1.28
+Added `check_config_against_spec` parameter to `Connector` abstract class 
+to allow skipping validating the input config against the spec for non-`check` calls
+
 ## 0.1.27
 Improving unit test for logger
 

diff --git a/airbyte-cdk/python/airbyte_cdk/connector.py b/airbyte-cdk/python/airbyte_cdk/connector.py
@@ -25,6 +25,9 @@ def __init__(self, spec_string):
 
 
 class Connector(ABC):
+    # configure whether the `check_config_against_spec_or_exit()` needs to be called
+    check_config_against_spec: bool = True
+
     # can be overridden to change an input config
     def configure(self, config: Mapping[str, Any], temp_dir: str) -> Mapping[str, Any]:
         """

diff --git a/airbyte-cdk/python/airbyte_cdk/destinations/destination.py b/airbyte-cdk/python/airbyte_cdk/destinations/destination.py
@@ -91,7 +91,8 @@ def run_cmd(self, parsed_args: argparse.Namespace) -> Iterable[AirbyteMessage]:
             yield AirbyteMessage(type=Type.SPEC, spec=spec)
             return
         config = self.read_config(config_path=parsed_args.config)
-        check_config_against_spec_or_exit(config, spec, self.logger)
+        if self.check_config_against_spec or cmd == "check":
+            check_config_against_spec_or_exit(config, spec, self.logger)
 
         if cmd == "check":
             yield self._run_check(config=config)

diff --git a/airbyte-cdk/python/airbyte_cdk/entrypoint.py b/airbyte-cdk/python/airbyte_cdk/entrypoint.py
@@ -72,7 +72,8 @@ def run(self, parsed_args: argparse.Namespace) -> Iterable[str]:
                 # Remove internal flags from config before validating so
                 # jsonschema's additionalProperties flag wont fail the validation
                 config, internal_config = split_config(config)
-                check_config_against_spec_or_exit(config, source_spec, self.logger)
+                if self.source.check_config_against_spec or cmd == "check":
+                    check_config_against_spec_or_exit(config, source_spec, self.logger)
                 # Put internal flags back to config dict
                 config.update(internal_config.dict())
 

diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py
@@ -15,7 +15,7 @@
 
 setup(
     name="airbyte-cdk",
-    version="0.1.27",
+    version="0.1.28",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",