airbyte-lib: Lightweight validation (airbytehq#34475)

airbyte-lib/README.md

@@ -24,3 +24,5 @@ airbyte-lib-validate-source —connector-dir . -—sample-config secrets/config.
 ```
 
 The script will install the python package in the provided directory, and run the connector against the provided config. The config should be a valid JSON file, with the same structure as the one that would be provided to the connector in Airbyte. The script will exit with a non-zero exit code if the connector fails to run.
+
+For a more lightweight check, the `--validate-install-only` flag can be used. This will only check that the connector can be installed and returns a spec, no sample config required.

airbyte-lib/airbyte_lib/_util/protocol_util.py

@@ -26,7 +26,7 @@ def airbyte_messages_to_record_dicts(
     yield from (
         cast(dict[str, Any], airbyte_message_to_record_dict(message))
         for message in messages
-        if message is not None
+        if message is not None and message.type == Type.RECORD
     )
 
 

airbyte-lib/airbyte_lib/validate.py

@@ -27,11 +27,16 @@ def _parse_args() -> argparse.Namespace:
         required=True,
         help="Path to the connector directory",
     )
+    parser.add_argument(
+        "--validate-install-only",
+        action="store_true",
+        help="Only validate that the connector can be installed and config can be validated.",
+    )
     parser.add_argument(
         "--sample-config",
         type=str,
-        required=True,
-        help="Path to the sample config.json file",
+        required=False,
+        help="Path to the sample config.json file. Required without --validate-install-only.",
     )
     return parser.parse_args()
 
@@ -45,7 +50,7 @@ def _run_subprocess_and_raise_on_failure(args: list[str]) -> None:
         )
 
 
-def tests(connector_name: str, sample_config: str) -> None:
+def full_tests(connector_name: str, sample_config: str) -> None:
     print("Creating source and validating spec and version...")
     source = ab.get_connector(
         # TODO: FIXME: noqa: SIM115, PTH123
@@ -76,6 +81,12 @@ def tests(connector_name: str, sample_config: str) -> None:
         )
 
 
+def install_only_test(connector_name: str) -> None:
+    print("Creating source and validating spec is returned successfully...")
+    source = ab.get_connector(connector_name)
+    source._get_spec(force_refresh=True)  # noqa: SLF001
+
+
 def run() -> None:
     """Handle CLI entrypoint for the `airbyte-lib-validate-source` command.
 
@@ -91,10 +102,11 @@ def run() -> None:
     args = _parse_args()
     connector_dir = args.connector_dir
     sample_config = args.sample_config
-    validate(connector_dir, sample_config)
+    validate_install_only = args.validate_install_only
+    validate(connector_dir, sample_config, validate_install_only=validate_install_only)
 
 
-def validate(connector_dir: str, sample_config: str) -> None:
+def validate(connector_dir: str, sample_config: str, *, validate_install_only: bool) -> None:
     # read metadata.yaml
     metadata_path = Path(connector_dir) / "metadata.yaml"
     with Path(metadata_path).open() as stream:
@@ -127,4 +139,9 @@ def validate(connector_dir: str, sample_config: str) -> None:
         temp_file.write(json.dumps(registry))
         temp_file.seek(0)
         os.environ["AIRBYTE_LOCAL_REGISTRY"] = str(temp_file.name)
-        tests(connector_name, sample_config)
+        if validate_install_only:
+            install_only_test(connector_name)
+        else:
+            if not sample_config:
+                raise Exception("sample_config is required when -validate-install-only is not set")
+            full_tests(connector_name, sample_config)

airbyte-lib/tests/integration_tests/fixtures/source-broken/metadata.yaml

@@ -0,0 +1,13 @@
+data:
+  connectorSubtype: api
+  connectorType: source
+  definitionId: 47f17145-fe20-4ef5-a548-e29b048adf84
+  dockerImageTag: 0.0.0
+  dockerRepository: airbyte/source-broken
+  githubIssueLabel: source-broken
+  name: Test
+  releaseDate: 2023-08-25
+  releaseStage: alpha
+  supportLevel: community
+  documentationUrl: https://docs.airbyte.com/integrations/sources/apify-dataset
+metadataSpecVersion: "1.0"

airbyte-lib/tests/integration_tests/fixtures/source-broken/setup.py

@@ -0,0 +1,20 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+from setuptools import find_packages, setup
+
+setup(
+    name="source_broken",
+    version="0.0.1",
+    description="Test Soutce",
+    author="Airbyte",
+    author_email="contact@airbyte.io",
+    packages=find_packages(),
+    entry_points={
+        "console_scripts": [
+            "source-broken=source_broken.run:run",
+        ],
+    },
+)

airbyte-lib/tests/integration_tests/fixtures/source-broken/source_broken/run.py

@@ -0,0 +1,4 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+def run():
+    raise Exception("Could not run")

airbyte-lib/tests/integration_tests/test_validation.py

@@ -8,8 +8,15 @@
 
 
 def test_validate_success():
-    validate("./tests/integration_tests/fixtures/source-test", "./tests/integration_tests/fixtures/valid_config.json")
+    validate("./tests/integration_tests/fixtures/source-test", "./tests/integration_tests/fixtures/valid_config.json", validate_install_only=False)
 
-def test_validate_failure():
+def test_validate_check_failure():
     with pytest.raises(Exception):
-        validate("./tests/integration_tests/fixtures/source-test", "./tests/integration_tests/fixtures/invalid_config.json")
+        validate("./tests/integration_tests/fixtures/source-test", "./tests/integration_tests/fixtures/invalid_config.json", validate_install_only=False)
+
+def test_validate_success_install_only():
+    validate("./tests/integration_tests/fixtures/source-test", "./tests/integration_tests/fixtures/invalid_config.json", validate_install_only=True)
+
+def test_validate_config_failure():
+    with pytest.raises(Exception):
+        validate("./tests/integration_tests/fixtures/source-broken", "./tests/integration_tests/fixtures/valid_config.json", validate_install_only=True)