add test that migration output schema same as source schema (#3356)

airbytehq · May 14, 2021 · 9b3b692 · 9b3b692
1 parent 0df5317
commit 9b3b692
Show file tree

Hide file tree

Showing 12 changed files with 333 additions and 35 deletions.
diff --git a/airbyte-migration/build.gradle b/airbyte-migration/build.gradle
@@ -6,6 +6,9 @@ dependencies {
     implementation project(':airbyte-json-validation')
 
     implementation 'net.sourceforge.argparse4j:argparse4j:0.8.1'
+
+    testImplementation project(':airbyte-config:models')
+    testImplementation project(':airbyte-db')
 }
 
 application {

diff --git a/airbyte-migration/src/main/java/io/airbyte/migrate/MigrationUtils.java b/airbyte-migration/src/main/java/io/airbyte/migrate/MigrationUtils.java
@@ -33,6 +33,7 @@
 import java.nio.file.Path;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -54,12 +55,18 @@ public class MigrationUtils {
    *        that are included here. resolving those dependencies is handled separately.
    * @return ResourceId to the JsonSchema found there.
    */
-  private static Map<ResourceId, JsonNode> getNameToSchemasFromPath(Path migrationResourcePath,
-                                                                    Path relativePath,
-                                                                    ResourceType resourceType,
-                                                                    Set<String> schemasToInclude) {
+  public static Map<ResourceId, JsonNode> getNameToSchemasFromResourcePath(Path migrationResourcePath,
+                                                                           Path relativePath,
+                                                                           ResourceType resourceType,
+                                                                           Set<String> schemasToInclude) {
+    return getNameToSchemasFromResourcePath(migrationResourcePath.resolve(relativePath), resourceType, schemasToInclude);
+  }
+
+  public static Map<ResourceId, JsonNode> getNameToSchemasFromResourcePath(Path pathToSchemasResource,
+                                                                           ResourceType resourceType,
+                                                                           Set<String> schemasToInclude) {
     final Map<ResourceId, JsonNode> schemas = new HashMap<>();
-    final Path pathToSchemas = JsonSchemas.prepareSchemas(migrationResourcePath.resolve(relativePath).toString(), MigrationUtils.class);
+    final Path pathToSchemas = JsonSchemas.prepareSchemas(pathToSchemasResource.toString(), MigrationUtils.class);
     FileUtils.listFiles(pathToSchemas.toFile(), null, false)
         .stream()
         .map(JsonSchemaValidator::getSchema)
@@ -72,16 +79,26 @@ private static Map<ResourceId, JsonNode> getNameToSchemasFromPath(Path migration
     return schemas;
   }
 
+  // this method is decently inefficient. if you need to fetch the schema for multiple configs, use
+  // getNameToSchemasFromResourcePath.
+  public static JsonNode getSchemaFromResourcePath(Path pathToSchema, ResourceId resourceId) {
+    final Map<ResourceId, JsonNode> nameToSchemas = getNameToSchemasFromResourcePath(
+        pathToSchema,
+        resourceId.getType(),
+        Collections.singleton(resourceId.getName()));
+    return nameToSchemas.get(resourceId);
+  }
+
   private static String getTitleAsConstantCase(JsonNode jsonNode) {
     return CaseFormat.UPPER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, jsonNode.get("title").asText());
   }
 
   public static Map<ResourceId, JsonNode> getConfigModels(Path migrationResourcePath, Set<String> schemasToInclude) {
-    return getNameToSchemasFromPath(migrationResourcePath, ResourceType.CONFIG.getDirectoryName(), ResourceType.CONFIG, schemasToInclude);
+    return getNameToSchemasFromResourcePath(migrationResourcePath, ResourceType.CONFIG.getDirectoryName(), ResourceType.CONFIG, schemasToInclude);
   }
 
   public static Map<ResourceId, JsonNode> getJobModels(Path migrationResourcePath, Set<String> schemasToInclude) {
-    return getNameToSchemasFromPath(migrationResourcePath, ResourceType.JOB.getDirectoryName(), ResourceType.JOB, schemasToInclude);
+    return getNameToSchemasFromResourcePath(migrationResourcePath, ResourceType.JOB.getDirectoryName(), ResourceType.JOB, schemasToInclude);
   }
 
   /**

diff --git a/airbyte-migration/src/main/java/io/airbyte/migrate/Migrations.java b/airbyte-migration/src/main/java/io/airbyte/migrate/Migrations.java
@@ -30,6 +30,7 @@
 import io.airbyte.migrate.migrations.MigrationV0_17_0;
 import io.airbyte.migrate.migrations.MigrationV0_18_0;
 import io.airbyte.migrate.migrations.MigrationV0_20_0;
+import io.airbyte.migrate.migrations.MigrationV0_23_0;
 import io.airbyte.migrate.migrations.NoOpMigration;
 import java.util.List;
 
@@ -45,6 +46,7 @@ public class Migrations {
   private static final Migration MIGRATION_V_0_20_0 = new MigrationV0_20_0(MIGRATION_V_0_19_0);
   private static final Migration MIGRATION_V_0_21_0 = new NoOpMigration(MIGRATION_V_0_20_0, "0.21.0-alpha");
   private static final Migration MIGRATION_V_0_22_0 = new NoOpMigration(MIGRATION_V_0_21_0, "0.22.0-alpha");
+  private static final Migration MIGRATION_V_0_23_0 = new MigrationV0_23_0(MIGRATION_V_0_22_0);
 
   // all migrations must be added to the list in the order that they should be applied.
   public static final List<Migration> MIGRATIONS = ImmutableList.of(
@@ -57,6 +59,7 @@ public class Migrations {
       MIGRATION_V_0_19_0,
       MIGRATION_V_0_20_0,
       MIGRATION_V_0_21_0,
-      MIGRATION_V_0_22_0);
+      MIGRATION_V_0_22_0,
+      MIGRATION_V_0_23_0);
 
 }
diff --git a/airbyte-migration/src/main/java/io/airbyte/migrate/ResourceId.java b/airbyte-migration/src/main/java/io/airbyte/migrate/ResourceId.java
@@ -84,6 +84,10 @@ public Path getResourceRelativePath() {
     return type.getDirectoryName().resolve(name + ".yaml");
   }
 
+  public String getName() {
+    return name;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {

diff --git a/airbyte-migration/src/main/java/io/airbyte/migrate/migrations/MigrationV0_14_3.java b/airbyte-migration/src/main/java/io/airbyte/migrate/migrations/MigrationV0_14_3.java
@@ -29,12 +29,12 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.resources.MoreResources;
 import io.airbyte.commons.util.MoreIterators;
 import io.airbyte.migrate.Migration;
+import io.airbyte.migrate.MigrationUtils;
 import io.airbyte.migrate.ResourceId;
 import io.airbyte.migrate.ResourceType;
-import java.io.IOException;
+import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -65,14 +65,9 @@ public String getVersion() {
   @Override
   public Map<ResourceId, JsonNode> getOutputSchema() {
     final Map<ResourceId, JsonNode> outputSchema = new HashMap<>(new MigrationV0_14_0().getOutputSchema());
-
-    try {
-      outputSchema.put(STANDARD_SYNC_RESOURCE_ID,
-          Jsons.jsonNode(MoreResources.readResource("migrations/migrationV0_14_0/airbyte_config/StandardSync.yaml")));
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-
+    outputSchema.put(
+        STANDARD_SYNC_RESOURCE_ID,
+        MigrationUtils.getSchemaFromResourcePath(Path.of("migrations/migrationV0_14_3/airbyte_config"), STANDARD_SYNC_RESOURCE_ID));
     return outputSchema;
   }
 

diff --git a/airbyte-migration/src/main/java/io/airbyte/migrate/migrations/MigrationV0_18_0.java b/airbyte-migration/src/main/java/io/airbyte/migrate/migrations/MigrationV0_18_0.java
@@ -28,12 +28,12 @@
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.ImmutableMap;
 import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.resources.MoreResources;
 import io.airbyte.commons.util.MoreIterators;
 import io.airbyte.migrate.Migration;
+import io.airbyte.migrate.MigrationUtils;
 import io.airbyte.migrate.ResourceId;
 import io.airbyte.migrate.ResourceType;
-import java.io.IOException;
+import java.nio.file.Path;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -78,12 +78,9 @@ public String getVersion() {
   @Override
   public Map<ResourceId, JsonNode> getInputSchema() {
     final Map<ResourceId, JsonNode> outputSchema = new HashMap<>(previousMigration.getOutputSchema());
-    try {
-      outputSchema.put(STANDARD_WORKSPACE_RESOURCE_ID,
-          Jsons.jsonNode(MoreResources.readResource("migrations/migrationV0_18_0/StandardWorkspace.yaml")));
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
+    outputSchema.put(
+        STANDARD_WORKSPACE_RESOURCE_ID,
+        MigrationUtils.getSchemaFromResourcePath(Path.of("migrations/migrationV0_18_0"), STANDARD_WORKSPACE_RESOURCE_ID));
     return outputSchema;
   }
 

diff --git a/airbyte-migration/src/main/java/io/airbyte/migrate/migrations/MigrationV0_20_0.java b/airbyte-migration/src/main/java/io/airbyte/migrate/migrations/MigrationV0_20_0.java
@@ -25,12 +25,11 @@
 package io.airbyte.migrate.migrations;
 
 import com.fasterxml.jackson.databind.JsonNode;
-import io.airbyte.commons.json.Jsons;
-import io.airbyte.commons.resources.MoreResources;
 import io.airbyte.migrate.Migration;
+import io.airbyte.migrate.MigrationUtils;
 import io.airbyte.migrate.ResourceId;
 import io.airbyte.migrate.ResourceType;
-import java.io.IOException;
+import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Consumer;
@@ -67,12 +66,9 @@ public String getVersion() {
   @Override
   public Map<ResourceId, JsonNode> getInputSchema() {
     final Map<ResourceId, JsonNode> outputSchema = new HashMap<>(previousMigration.getOutputSchema());
-    try {
-      outputSchema.put(STANDARD_WORKSPACE_RESOURCE_ID,
-          Jsons.jsonNode(MoreResources.readResource("migrations/migrationV0_20_0/StandardWorkspace.yaml")));
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
+    outputSchema.put(
+        STANDARD_WORKSPACE_RESOURCE_ID,
+        MigrationUtils.getSchemaFromResourcePath(Path.of("migrations/migrationV0_20_0"), STANDARD_WORKSPACE_RESOURCE_ID));
     return outputSchema;
   }
 

diff --git a/airbyte-migration/src/main/java/io/airbyte/migrate/migrations/MigrationV0_23_0.java b/airbyte-migration/src/main/java/io/airbyte/migrate/migrations/MigrationV0_23_0.java
@@ -0,0 +1,101 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.migrate.migrations;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.airbyte.migrate.Migration;
+import io.airbyte.migrate.MigrationUtils;
+import io.airbyte.migrate.ResourceId;
+import io.airbyte.migrate.ResourceType;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This migration is currently empty and is a placeholder for migrations for the next 0.19.0 release
+ *
+ * Additionally, this migration updates the JSON Schema for StandardWorkspace with a new optional
+ * field 'failureNotificationsWebhook' introduced in issue #1689
+ */
+public class MigrationV0_23_0 extends BaseMigration implements Migration {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(MigrationV0_23_0.class);
+
+  private static final ResourceId SOURCE_DEFINITION_RESOURCE_ID = ResourceId.fromConstantCase(ResourceType.CONFIG, "STANDARD_SOURCE_DEFINITION");
+  private static final ResourceId DESTINATION_DEFINITION_RESOURCE_ID =
+      ResourceId.fromConstantCase(ResourceType.CONFIG, "STANDARD_DESTINATION_DEFINITION");
+  private static final ResourceId STANDARD_SYNC_RESOURCE_ID = ResourceId.fromConstantCase(ResourceType.CONFIG, "STANDARD_SYNC");
+
+  private static final String MIGRATION_VERSION = "0.23.0-alpha";
+
+  private final Migration previousMigration;
+
+  public MigrationV0_23_0(Migration previousMigration) {
+    super(previousMigration);
+    this.previousMigration = previousMigration;
+  }
+
+  @Override
+  public String getVersion() {
+    return MIGRATION_VERSION;
+  }
+
+  @Override
+  public Map<ResourceId, JsonNode> getInputSchema() {
+    return new HashMap<>(previousMigration.getOutputSchema());
+  }
+
+  @Override
+  public Map<ResourceId, JsonNode> getOutputSchema() {
+    final Map<ResourceId, JsonNode> outputSchema = new HashMap<>(previousMigration.getOutputSchema());
+    outputSchema.put(
+        SOURCE_DEFINITION_RESOURCE_ID,
+        MigrationUtils.getSchemaFromResourcePath(Path.of("migrations/migrationV0_23_0"), SOURCE_DEFINITION_RESOURCE_ID));
+    outputSchema.put(
+        DESTINATION_DEFINITION_RESOURCE_ID,
+        MigrationUtils.getSchemaFromResourcePath(Path.of("migrations/migrationV0_23_0"), DESTINATION_DEFINITION_RESOURCE_ID));
+    outputSchema.put(
+        STANDARD_SYNC_RESOURCE_ID,
+        MigrationUtils.getSchemaFromResourcePath(Path.of("migrations/migrationV0_23_0"), STANDARD_SYNC_RESOURCE_ID));
+    return outputSchema;
+  }
+
+  @Override
+  public void migrate(Map<ResourceId, Stream<JsonNode>> inputData, Map<ResourceId, Consumer<JsonNode>> outputData) {
+    for (final Map.Entry<ResourceId, Stream<JsonNode>> entry : inputData.entrySet()) {
+      final Consumer<JsonNode> recordConsumer = outputData.get(entry.getKey());
+
+      entry.getValue().forEach(r -> {
+        // empty migration
+        recordConsumer.accept(r);
+      });
+    }
+  }
+
+}
diff --git a/...gration/src/main/resources/migrations/migrationV0_23_0/StandardDestinationDefinition.yaml b/...gration/src/main/resources/migrations/migrationV0_23_0/StandardDestinationDefinition.yaml
@@ -0,0 +1,27 @@
+---
+"$schema": http://json-schema.org/draft-07/schema#
+"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/StandardDestinationDefinition.yaml
+title: StandardDestinationDefinition
+description: describes a destination
+type: object
+required:
+  - destinationDefinitionId
+  - name
+  - dockerRepository
+  - dockerImageTag
+  - documentationUrl
+additionalProperties: false
+properties:
+  destinationDefinitionId:
+    type: string
+    format: uuid
+  name:
+    type: string
+  dockerRepository:
+    type: string
+  dockerImageTag:
+    type: string
+  documentationUrl:
+    type: string
+  icon:
+    type: string
diff --git a/...te-migration/src/main/resources/migrations/migrationV0_23_0/StandardSourceDefinition.yaml b/...te-migration/src/main/resources/migrations/migrationV0_23_0/StandardSourceDefinition.yaml
@@ -0,0 +1,27 @@
+---
+"$schema": http://json-schema.org/draft-07/schema#
+"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/Source.yaml
+title: StandardSourceDefinition
+description: describes a source
+type: object
+required:
+  - sourceDefinitionId
+  - name
+  - dockerRepository
+  - dockerImageTag
+  - documentationUrl
+additionalProperties: false
+properties:
+  sourceDefinitionId:
+    type: string
+    format: uuid
+  name:
+    type: string
+  dockerRepository:
+    type: string
+  dockerImageTag:
+    type: string
+  documentationUrl:
+    type: string
+  icon:
+    type: string
diff --git a/airbyte-migration/src/main/resources/migrations/migrationV0_23_0/StandardSync.yaml b/airbyte-migration/src/main/resources/migrations/migrationV0_23_0/StandardSync.yaml
@@ -0,0 +1,35 @@
+---
+"$schema": http://json-schema.org/draft-07/schema#
+"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/StandardSync.yaml
+title: StandardSync
+description: configuration required for sync for ALL taps
+type: object
+required:
+  - sourceId
+  - destinationId
+  - name
+  - catalog
+additionalProperties: false
+properties:
+  prefix:
+    description: Prefix that will be prepended to the name of each stream when it is written to the destination.
+    type: string
+  sourceId:
+    type: string
+    format: uuid
+  destinationId:
+    type: string
+    format: uuid
+  connectionId:
+    type: string
+    format: uuid
+  name:
+    type: string
+  catalog:
+    existingJavaType: io.airbyte.protocol.models.ConfiguredAirbyteCatalog
+  status:
+    type: string
+    enum:
+      - active
+      - inactive
+      - deprecated