🎉 Added JDBC source connector bootstrap template (#3819)

* [3560] Added JDBC source connector bootstrap template

airbyte-integrations/connector-templates/generator/build.gradle

@@ -40,5 +40,6 @@ def addScaffoldTemplateTask(name, packageName,scaffoldParams=[]) {
 
 addScaffoldTemplateTask('Python Source', 'scaffold-source-python')
 addScaffoldTemplateTask('Python HTTP API Source', 'scaffold-source-http')
+addScaffoldTemplateTask('Java JDBC Source', 'scaffold-java-jdbc')
 // TODO: enable Singer template testing
 //addScaffoldTask('source-python-singer', ['tap-exchangeratesapi'])

airbyte-integrations/connector-templates/generator/plopfile.js

@@ -29,13 +29,15 @@ module.exports = function (plop) {
   const pythonSourceInputRoot = '../source-python';
   const singerSourceInputRoot = '../source-singer';
   const genericSourceInputRoot = '../source-generic';
+  const genericJdbcSourceInputRoot = '../source-java-jdbc';
   const httpApiInputRoot = '../source-python-http-api';
   const javaDestinationInput = '../destination-java';
 
   const outputDir = '../../connectors';
   const pythonSourceOutputRoot = `${outputDir}/source-{{dashCase name}}`;
   const singerSourceOutputRoot = `${outputDir}/source-{{dashCase name}}-singer`;
   const genericSourceOutputRoot = `${outputDir}/source-{{dashCase name}}`;
+  const genericJdbcSourceOutputRoot = `${outputDir}/source-{{dashCase name}}`;
   const httpApiOutputRoot = `${outputDir}/source-{{dashCase name}}`;
   const javaDestinationOutputRoot = `${outputDir}/destination-{{dashCase name}}`;
 
@@ -123,6 +125,21 @@ module.exports = function (plop) {
             {type: 'emitSuccess', outputPath: pythonSourceOutputRoot, message: "For a checklist of what to do next go to https://docs.airbyte.io/tutorials/building-a-python-source"}]
     });
 
+  plop.setGenerator('Java JDBC Source', {
+    description: 'Generate a minimal Java JDBC Airbyte Source Connector.',
+    prompts: [{type: 'input', name: 'name', message: 'Source name, without the "source-" prefix e.g: "mysql"'}],
+    actions: [
+      {
+        abortOnFail: true,
+        type:'addMany',
+        destination: genericJdbcSourceOutputRoot,
+        base: genericJdbcSourceInputRoot,
+        templateFiles: `${genericJdbcSourceInputRoot}/**/**`,
+      },
+      {type: 'emitSuccess', outputPath: genericJdbcSourceOutputRoot}
+    ]
+  });
+
   plop.setGenerator('Generic Source', {
       description: 'Use if none of the other templates apply to your use case.',
       prompts: [{type: 'input', name: 'name', message: 'Source name, without the "source-" prefix e.g: "google-analytics"'}],

airbyte-integrations/connector-templates/source-java-jdbc/.dockerignore

@@ -0,0 +1,3 @@
+*
+!Dockerfile
+!build

airbyte-integrations/connector-templates/source-java-jdbc/Dockerfile

@@ -0,0 +1,13 @@
+FROM airbyte/integration-base-java:dev
+
+WORKDIR /airbyte
+
+ENV APPLICATION source-{{dashCase name}}
+
+COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
+
+RUN tar xf ${APPLICATION}.tar --strip-components=1
+
+# Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile.
+LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.name=airbyte/source-{{dashCase name}}

airbyte-integrations/connector-templates/source-java-jdbc/build.gradle

@@ -0,0 +1,28 @@
+plugins {
+    id 'application'
+    id 'airbyte-docker'
+    id 'airbyte-integration-test-java'
+}
+
+application {
+    mainClass = 'io.airbyte.integrations.source.{{dashCase name}}.{{pascalCase name}}Source'
+}
+
+dependencies {
+    implementation project(':airbyte-db')
+    implementation project(':airbyte-integrations:bases:base-java')
+    implementation project(':airbyte-protocol:models')
+    implementation project(':airbyte-integrations:connectors:source-jdbc')
+
+    //TODO Add jdbc driver import here. Ex: implementation 'com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre14'
+
+    testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc'))
+
+    testImplementation 'org.apache.commons:commons-lang3:3.11'
+
+    integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-{{dashCase name}}')
+    integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test')
+
+    implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
+    integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
+}

airbyte-integrations/connector-templates/source-java-jdbc/src/main/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs

@@ -0,0 +1,70 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.integrations.source.{{snakeCase name}};
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.airbyte.db.jdbc.NoOpJdbcStreamingQueryConfiguration;
+import io.airbyte.integrations.base.IntegrationRunner;
+import io.airbyte.integrations.base.Source;
+import io.airbyte.integrations.source.jdbc.AbstractJdbcSource;
+import java.util.Set;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class {{pascalCase name}}Source extends AbstractJdbcSource implements Source {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger({{pascalCase name}}Source.class);
+
+  // TODO insert your driver name. Ex: "com.microsoft.sqlserver.jdbc.SQLServerDriver"
+  static final String DRIVER_CLASS = "driver_name_here";
+
+  public {{pascalCase name}}Source() {
+    // By default NoOpJdbcStreamingQueryConfiguration class is used, but may be updated. See see example
+    // MssqlJdbcStreamingQueryConfiguration
+    super(DRIVER_CLASS, new NoOpJdbcStreamingQueryConfiguration());
+  }
+
+  // TODO The config is based on spec.json, update according to your DB
+  @Override
+  public JsonNode toJdbcConfig(JsonNode aqqConfig) {
+    // TODO create DB config. Ex: "Jsons.jsonNode(ImmutableMap.builder().put("username",
+    // userName).put("password", pas)...build());
+    return null;
+  }
+
+  @Override
+  public Set<String> getExcludedInternalSchemas() {
+    // TODO Add tables to exaclude, Ex "INFORMATION_SCHEMA", "sys", "spt_fallback_db", etc
+    return Set.of("");
+  }
+
+  public static void main(String[] args) throws Exception {
+    final Source source = new {{pascalCase name}}Source();
+    LOGGER.info("starting source: {}", {{pascalCase name}}Source.class);
+    new IntegrationRunner(source).run(args);
+    LOGGER.info("completed source: {}", {{pascalCase name}}Source.class);
+  }
+
+}

airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs

@@ -0,0 +1,55 @@
+{
+  "documentationUrl": "https://docs.airbyte.io/integrations/source/mysql",
+  "connectionSpecification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "{{pascalCase name}} Source Spec",
+    "type": "object",
+    "required": ["host", "port", "database", "username", "replication_method"],
+    "additionalProperties": false,
+    "properties": {
+      "host": {
+        "description": "Hostname of the database.",
+        "type": "string",
+        "order": 0
+      },
+      "port": {
+        "description": "Port of the database.",
+        "type": "integer",
+        "minimum": 0,
+        "maximum": 65536,
+        "default": 3306,
+        "examples": ["3306"],
+        "order": 1
+      },
+      "database": {
+        "description": "Name of the database.",
+        "type": "string",
+        "order": 2
+      },
+      "username": {
+        "description": "Username to use to access the database.",
+        "type": "string",
+        "order": 3
+      },
+      "password": {
+        "description": "Password associated with the username.",
+        "type": "string",
+        "airbyte_secret": true,
+        "order": 4
+      },
+      "jdbc_url_params": {
+        "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)",
+        "type": "string",
+        "order": 5
+      },
+      "replication_method": {
+        "type": "string",
+        "title": "Replication Method",
+        "description": "Replication method to use for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.",
+        "order": 6,
+        "default": "STANDARD",
+        "enum": ["STANDARD", "CDC"]
+      }
+    }
+  }
+}

airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs

@@ -0,0 +1,87 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.integrations.source.{{snakeCase name}};
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.resources.MoreResources;
+import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest;
+import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
+import io.airbyte.protocol.models.ConnectorSpecification;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+
+public class {{pascalCase name}}SourceAcceptanceTest extends SourceAcceptanceTest {
+
+  private JsonNode config;
+
+  @Override
+  protected void setup(TestDestinationEnv testEnv) throws Exception {
+    // TODO create new container. Ex: "new OracleContainer("epiclabs/docker-oracle-xe-11g");"
+    // TODO make container started. Ex: "container.start();"
+    // TODO init JsonNode config
+    // TODO crete airbyte Database object "Databases.createJdbcDatabase(...)"
+    // TODO insert test data to DB. Ex: "database.execute(connection-> ...)"
+    // TODO close Database. Ex: "database.close();"
+  }
+
+  @Override
+  protected void tearDown(TestDestinationEnv testEnv) {
+    // TODO close container that was initialized in setup() method. Ex: "container.close();"
+  }
+
+  @Override
+  protected String getImageName() {
+    return "airbyte/source-{{dashCase name}}:dev";
+  }
+
+  @Override
+  protected ConnectorSpecification getSpec() throws Exception {
+    return Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class);
+  }
+
+  @Override
+  protected JsonNode getConfig() {
+    return config;
+  }
+
+  @Override
+  protected ConfiguredAirbyteCatalog getConfiguredCatalog() {
+    // TODO Return the ConfiguredAirbyteCatalog with ConfiguredAirbyteStream objects
+    return null;
+  }
+
+  @Override
+  protected List<String> getRegexTests() {
+    return Collections.emptyList();
+  }
+
+  @Override
+  protected JsonNode getState() {
+    return Jsons.jsonNode(new HashMap<>());
+  }
+
+}

airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}JdbcSourceAcceptanceTest.java.hbs

@@ -0,0 +1,88 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.integrations.source.{{snakeCase name}};
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.airbyte.integrations.source.jdbc.AbstractJdbcSource;
+import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class {{pascalCase name}}JdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger({{pascalCase name}}JdbcSourceAcceptanceTest.class);
+
+  // TODO declare a test container for DB. EX: org.testcontainers.containers.OracleContainer
+
+  @BeforeAll
+  static void init() {
+    // Oracle returns uppercase values
+    // TODO init test container. Ex: "new OracleContainer("epiclabs/docker-oracle-xe-11g")"
+    // TODO start container. Ex: "container.start();"
+  }
+
+  @BeforeEach
+  public void setup() throws Exception {
+    // TODO init config. Ex: "config = Jsons.jsonNode(ImmutableMap.builder().put("host",
+    // host).put("port", port)....build());
+    super.setup();
+  }
+
+  @AfterEach
+  public void tearDown() {
+    // TODO clean used resources
+  }
+
+  @Override
+  public AbstractJdbcSource getSource() {
+    return new {{pascalCase name}}Source();
+  }
+
+  @Override
+  public boolean supportsSchemas() {
+    // TODO check if your db supports it and update method accordingly
+    return false;
+  }
+
+  @Override
+  public JsonNode getConfig() {
+    return config;
+  }
+
+  @Override
+  public String getDriverClass() {
+    return {{pascalCase name}}Source.DRIVER_CLASS;
+  }
+
+  @AfterAll
+  static void cleanUp() {
+    // TODO close the container. Ex: "container.close();"
+  }
+
+}