🚨🚨 e2e-test: Standardize e2e testing destination spec (#25776)

* Standardize e2e testing destination * Change casing * Fix implementation * Update versions and docs * Do major version bump * Update acceptance tests * use correct name * Fix implementation * Automated Change * Revert "Automated Change" This reverts commit 816b662. * Automated Change * Fix unit test --------- Co-authored-by: terencecho <terencecho@users.noreply.github.com>
airbytehq · May 15, 2023 · 780903f · 780903f
1 parent d0fa40c
commit 780903f
Show file tree

Hide file tree

Showing 18 changed files with 254 additions and 191 deletions.
diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/destination_definitions.yaml b/airbyte-config-oss/init-oss/src/main/resources/seed/destination_definitions.yaml
@@ -140,7 +140,7 @@
 - name: E2E Testing
   destinationDefinitionId: 2eb65e87-983a-4fd7-b3e3-9d9dc6eb8537
   dockerRepository: airbyte/destination-e2e-test
-  dockerImageTag: 0.2.4
+  dockerImageTag: 0.3.0
   documentationUrl: https://docs.airbyte.com/integrations/destinations/e2e-test
   icon: airbyte.svg
 - destinationDefinitionId: 68f351a7-2745-4bef-ad7f-996b8e51bb8c

diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/destination_specs.yaml b/airbyte-config-oss/init-oss/src/main/resources/seed/destination_specs.yaml
@@ -2284,7 +2284,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-e2e-test:0.2.4"
+- dockerImage: "airbyte/destination-e2e-test:0.3.0"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/destinations/e2e-test"
     connectionSpecification:

diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json b/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json
@@ -2097,7 +2097,7 @@
     "destinationDefinitionId": "2eb65e87-983a-4fd7-b3e3-9d9dc6eb8537",
     "name": "E2E Testing",
     "dockerRepository": "airbyte/destination-e2e-test",
-    "dockerImageTag": "0.2.4",
+    "dockerImageTag": "0.3.0",
     "documentationUrl": "https://docs.airbyte.com/integrations/destinations/e2e-test",
     "icon": "airbyte.svg",
     "spec": {

diff --git a/airbyte-integrations/connectors/destination-dev-null/Dockerfile b/airbyte-integrations/connectors/destination-dev-null/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION destination-dev-null
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.2.7
+LABEL io.airbyte.version=0.3.0
 LABEL io.airbyte.name=airbyte/destination-dev-null
diff --git a/airbyte-integrations/connectors/destination-dev-null/metadata.yaml b/airbyte-integrations/connectors/destination-dev-null/metadata.yaml
@@ -2,7 +2,7 @@ data:
   connectorSubtype: file
   connectorType: destination
   definitionId: a7bcc9d8-13b3-4e49-b80d-d020b90045e3
-  dockerImageTag: 0.2.7
+  dockerImageTag: 0.3.0
   dockerRepository: airbyte/destination-dev-null
   githubIssueLabel: destination-dev-null
   license: MIT

diff --git a/...v-null/src/main/java/io/airbyte/integrations/destination/dev_null/DevNullDestination.java b/...v-null/src/main/java/io/airbyte/integrations/destination/dev_null/DevNullDestination.java
@@ -41,11 +41,12 @@ public ConnectorSpecification modifySpec(final ConnectorSpecification originalSp
 
     ((ObjectNode) spec.getConnectionSpecification()).put("title", DEV_NULL_DESTINATION_TITLE);
 
-    final ArrayNode types = (ArrayNode) spec.getConnectionSpecification().get("oneOf");
+    final ObjectNode properties = (ObjectNode) spec.getConnectionSpecification().get("test_destination");
+    final ArrayNode types = (ArrayNode) properties.get("test_destination_type").get("oneOf");
     final Iterator<JsonNode> typesIterator = types.elements();
     while (typesIterator.hasNext()) {
       final JsonNode typeNode = typesIterator.next();
-      if (!typeNode.get("properties").get("type").get("const").asText().equalsIgnoreCase("silent")) {
+      if (!typeNode.get("test_destination").get("test_destination_type").get("const").asText().equalsIgnoreCase("silent")) {
         typesIterator.remove();
       }
     }

diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-dev-null/src/test/resources/expected_spec.json
@@ -4,22 +4,31 @@
   "supportsNormalization": false,
   "supportsDBT": false,
   "supported_destination_sync_modes": ["overwrite", "append"],
+  "protocol_version": "0.2.1",
   "connectionSpecification": {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "E2E Test (/dev/null) Destination Spec",
     "type": "object",
-    "oneOf": [
-      {
-        "title": "Silent",
-        "required": ["type"],
-        "properties": {
-          "type": {
-            "type": "string",
-            "const": "SILENT",
-            "default": "SILENT"
+    "required": ["test_destination"],
+    "properties": {
+      "test_destination": {
+        "title" : "Test Destination",
+        "type" : "object",
+        "description" : "The type of destination to be used",
+        "oneOf": [
+          {
+            "title": "Silent",
+            "required": ["test_destination_type"],
+            "properties": {
+              "test_destination_type": {
+                "type": "string",
+                "const": "SILENT",
+                "default": "SILENT"
+              }
+            }
           }
-        }
+        ]
       }
-    ]
+    }
   }
 }
diff --git a/airbyte-integrations/connectors/destination-e2e-test/Dockerfile b/airbyte-integrations/connectors/destination-e2e-test/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION destination-e2e-test
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.2.4
+LABEL io.airbyte.version=0.3.0
 LABEL io.airbyte.name=airbyte/destination-e2e-test
diff --git a/airbyte-integrations/connectors/destination-e2e-test/metadata.yaml b/airbyte-integrations/connectors/destination-e2e-test/metadata.yaml
@@ -2,7 +2,7 @@ data:
   connectorSubtype: unknown
   connectorType: destination
   definitionId: 2eb65e87-983a-4fd7-b3e3-9d9dc6eb8537
-  dockerImageTag: 0.2.4
+  dockerImageTag: 0.3.0
   dockerRepository: airbyte/destination-e2e-test
   githubIssueLabel: destination-e2e-test
   icon: airbyte.svg

diff --git a/...est/src/main/java/io/airbyte/integrations/destination/e2e_test/FailAfterNDestination.java b/...est/src/main/java/io/airbyte/integrations/destination/e2e_test/FailAfterNDestination.java
@@ -30,7 +30,7 @@ public AirbyteConnectionStatus check(final JsonNode config) {
   public AirbyteMessageConsumer getConsumer(final JsonNode config,
                                             final ConfiguredAirbyteCatalog catalog,
                                             final Consumer<AirbyteMessage> outputRecordCollector) {
-    return new FailAfterNConsumer(config.get("num_messages").asLong(), outputRecordCollector);
+    return new FailAfterNConsumer(config.get("test_destination").get("num_messages").asLong(), outputRecordCollector);
   }
 
   public static class FailAfterNConsumer implements AirbyteMessageConsumer {

diff --git a/...-test/src/main/java/io/airbyte/integrations/destination/e2e_test/TestingDestinations.java b/...-test/src/main/java/io/airbyte/integrations/destination/e2e_test/TestingDestinations.java
@@ -46,7 +46,7 @@ public TestingDestinations(final Map<TestDestinationType, Destination> destinati
   }
 
   private Destination selectDestination(final JsonNode config) {
-    return destinationMap.get(TestDestinationType.valueOf(config.get("type").asText()));
+    return destinationMap.get(TestDestinationType.valueOf(config.get("test_destination").get("test_destination_type").asText()));
   }
 
   @Override

diff --git a/...test/src/main/java/io/airbyte/integrations/destination/e2e_test/ThrottledDestination.java b/...test/src/main/java/io/airbyte/integrations/destination/e2e_test/ThrottledDestination.java
@@ -36,7 +36,7 @@ public AirbyteConnectionStatus check(final JsonNode config) {
   public AirbyteMessageConsumer getConsumer(final JsonNode config,
                                             final ConfiguredAirbyteCatalog catalog,
                                             final Consumer<AirbyteMessage> outputRecordCollector) {
-    return new ThrottledConsumer(config.get("millis_per_record").asLong(), outputRecordCollector);
+    return new ThrottledConsumer(config.get("test_destination").get("millis_per_record").asLong(), outputRecordCollector);
   }
 
   public static class ThrottledConsumer implements AirbyteMessageConsumer {

diff --git a/.../main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLoggerFactory.java b/.../main/java/io/airbyte/integrations/destination/e2e_test/logging/TestingLoggerFactory.java
@@ -17,11 +17,17 @@ public TestingLoggerFactory(final JsonNode config) {
   }
 
   public TestingLogger create(final AirbyteStreamNameNamespacePair streamNamePair) {
-    if (!config.has("logging_config")) {
+    if (!config.has("test_destination")) {
+      throw new IllegalArgumentException("Property test_destination is required, but not found");
+    }
+
+    final JsonNode testDestinationConfig = config.get("test_destination");
+
+    if (!testDestinationConfig.has("logging_config")) {
       throw new IllegalArgumentException("Property logging_config is required, but not found");
     }
 
-    final JsonNode logConfig = config.get("logging_config");
+    final JsonNode logConfig = testDestinationConfig.get("logging_config");
     final LoggingType loggingType = LoggingType.valueOf(logConfig.get("logging_type").asText());
     switch (loggingType) {
       case FirstN -> {