diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/.gitignore b/airbyte-integrations/connectors/source-db2-strict-encrypt/.gitignore new file mode 100644 index 0000000000000..65c7d0ad3e73c --- /dev/null +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/.gitignore @@ -0,0 +1,3 @@ +* +!Dockerfile +!build diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile new file mode 100644 index 0000000000000..21e649ceebc27 --- /dev/null +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile @@ -0,0 +1,12 @@ +FROM airbyte/integration-base-java:dev + +WORKDIR /airbyte + +ENV APPLICATION source-db2-strict-encrypt + +COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar + +RUN tar xf ${APPLICATION}.tar --strip-components=1 + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-db2-strict-encrypt diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/acceptance-test-config.yml b/airbyte-integrations/connectors/source-db2-strict-encrypt/acceptance-test-config.yml new file mode 100644 index 0000000000000..14f479009a009 --- /dev/null +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/acceptance-test-config.yml @@ -0,0 +1,6 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-db2-strict-encrypt:dev +tests: + spec: + - spec_path: "src/test/resources/expected_spec.json" diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-db2-strict-encrypt/build.gradle new file mode 100644 index 0000000000000..2a0451a2ec846 --- /dev/null +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/build.gradle @@ -0,0 +1,31 @@ +plugins { + id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' +} + +application { + mainClass = 'io.airbyte.integrations.source.db2_strict_encrypt.Db2StrictEncryptSource' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] +} + +dependencies { + implementation project(':airbyte-db:lib') + implementation project(':airbyte-integrations:connectors:source-db2') + implementation project(':airbyte-integrations:bases:base-java') + implementation project(':airbyte-integrations:connectors:source-jdbc') + implementation project(':airbyte-integrations:connectors:source-relational-db') + implementation project(':airbyte-protocol:models') + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + + implementation group: 'com.ibm.db2', name: 'jcc', version: '11.5.5.0' + + testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) + testImplementation project(':airbyte-test-utils') + testImplementation "org.testcontainers:db2:1.15.3" + + integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') + integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-db2') + integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + integrationTestJavaImplementation 'org.apache.commons:commons-lang3:3.11' +} diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/main/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2JdbcStreamingQueryConfiguration.java b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/main/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2JdbcStreamingQueryConfiguration.java new file mode 100644 index 0000000000000..71c7c3d00ca02 --- /dev/null +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/main/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2JdbcStreamingQueryConfiguration.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.db2_strict_encrypt; + +import io.airbyte.db.jdbc.JdbcStreamingQueryConfiguration; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +public class Db2JdbcStreamingQueryConfiguration implements + JdbcStreamingQueryConfiguration { + + @Override + public void accept(final Connection connection, final PreparedStatement preparedStatement) + throws SQLException { + connection.setAutoCommit(false); + preparedStatement.setFetchSize(1000); + } + +} diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/main/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2StrictEncryptSource.java b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/main/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2StrictEncryptSource.java new file mode 100644 index 0000000000000..e90d751cf02ef --- /dev/null +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/main/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2StrictEncryptSource.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.db2_strict_encrypt; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.integrations.base.Source; +import io.airbyte.integrations.base.spec_modification.SpecModifyingSource; +import io.airbyte.integrations.source.db2.Db2Source; +import io.airbyte.protocol.models.ConnectorSpecification; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Db2StrictEncryptSource extends SpecModifyingSource implements Source { + + private static final Logger LOGGER = LoggerFactory.getLogger(Db2StrictEncryptSource.class); + public static final String DRIVER_CLASS = "com.ibm.db2.jcc.DB2Driver"; + + public Db2StrictEncryptSource() { + super(new Db2Source()); + } + + @Override + public ConnectorSpecification modifySpec(final ConnectorSpecification originalSpec) { + final ConnectorSpecification spec = Jsons.clone(originalSpec); + // We need to remove the first item from one Of, which is responsible for connecting to the source + // without encrypted. + ((ArrayNode) spec.getConnectionSpecification().get("properties").get("encryption").get("oneOf")).remove(0); + return spec; + } + + public static void main(final String[] args) throws Exception { + final Source source = new Db2StrictEncryptSource(); + LOGGER.info("starting source: {}", Db2StrictEncryptSource.class); + new IntegrationRunner(source).run(args); + LOGGER.info("completed source: {}", Db2StrictEncryptSource.class); + } + +} diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java new file mode 100644 index 0000000000000..3731c00630583 --- /dev/null +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.db.Databases; +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.source.db2.Db2Source; +import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.protocol.models.CatalogHelpers; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaPrimitive; +import io.airbyte.protocol.models.SyncMode; +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.testcontainers.containers.Db2Container; + +public class Db2StrictEncryptSourceCertificateAcceptanceTest extends SourceAcceptanceTest { + + private static final String SCHEMA_NAME = "SOURCE_INTEGRATION_TEST"; + private static final String STREAM_NAME1 = "ID_AND_NAME1"; + private static final String STREAM_NAME2 = "ID_AND_NAME2"; + + private static final String TEST_KEY_STORE_PASS = "Passw0rd"; + private static final String KEY_STORE_FILE_PATH = "clientkeystore.jks"; + private static final String SSL_CONFIG = ":sslConnection=true;sslTrustStoreLocation=" + KEY_STORE_FILE_PATH + + ";sslTrustStorePassword=" + TEST_KEY_STORE_PASS + ";"; + + private Db2Container db; + private JsonNode config; + private JdbcDatabase database; + + @Override + protected String getImageName() { + return "airbyte/source-db2-strict-encrypt:dev"; + } + + @Override + protected ConnectorSpecification getSpec() throws Exception { + return Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class); + } + + @Override + protected JsonNode getConfig() { + return config; + } + + @Override + protected ConfiguredAirbyteCatalog getConfiguredCatalog() { + return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(Lists.newArrayList("ID")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + String.format("%s.%s", SCHEMA_NAME, STREAM_NAME1), + Field.of("ID", JsonSchemaPrimitive.NUMBER), + Field.of("NAME", JsonSchemaPrimitive.STRING)) + .withSupportedSyncModes( + Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))), + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.FULL_REFRESH) + .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) + .withStream(CatalogHelpers.createAirbyteStream( + String.format("%s.%s", SCHEMA_NAME, STREAM_NAME2), + Field.of("ID", JsonSchemaPrimitive.NUMBER), + Field.of("NAME", JsonSchemaPrimitive.STRING)) + .withSupportedSyncModes( + Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); + } + + @Override + protected JsonNode getState() { + return Jsons.jsonNode(new HashMap<>()); + } + + @Override + protected List getRegexTests() { + return Collections.emptyList(); + } + + @Override + protected void setupEnvironment(TestDestinationEnv environment) throws Exception { + db = new Db2Container("ibmcom/db2:11.5.5.0").withCommand().acceptLicense() + .withExposedPorts(50000); + db.start(); + + var certificate = getCertificate(); + try { + convertAndImportCertificate(certificate); + } catch (IOException | InterruptedException e) { + throw new RuntimeException("Failed to import certificate into Java Keystore"); + } + + config = Jsons.jsonNode(ImmutableMap.builder() + .put("host", db.getHost()) + .put("port", db.getMappedPort(50000)) + .put("db", db.getDatabaseName()) + .put("username", db.getUsername()) + .put("password", db.getPassword()) + .put("encryption", Jsons.jsonNode(ImmutableMap.builder() + .put("encryption_method", "encrypted_verify_certificate") + .put("ssl_certificate", certificate) + .put("key_store_password", TEST_KEY_STORE_PASS) + .build())) + .build()); + + String jdbcUrl = String.format("jdbc:db2://%s:%s/%s", + config.get("host").asText(), + db.getMappedPort(50000), + config.get("db").asText()) + SSL_CONFIG; + + database = Databases.createJdbcDatabase( + config.get("username").asText(), + config.get("password").asText(), + jdbcUrl, + Db2Source.DRIVER_CLASS); + + final String createSchemaQuery = String.format("CREATE SCHEMA %s", SCHEMA_NAME); + final String createTableQuery1 = String + .format("CREATE TABLE %s.%s (ID INTEGER, NAME VARCHAR(200))", SCHEMA_NAME, STREAM_NAME1); + final String createTableQuery2 = String + .format("CREATE TABLE %s.%s (ID INTEGER, NAME VARCHAR(200))", SCHEMA_NAME, STREAM_NAME2); + final String insertIntoTableQuery1 = String + .format("INSERT INTO %s.%s (ID, NAME) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash')", + SCHEMA_NAME, STREAM_NAME1); + final String insertIntoTableQuery2 = String + .format("INSERT INTO %s.%s (ID, NAME) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash')", + SCHEMA_NAME, STREAM_NAME2); + + database.execute(createSchemaQuery); + database.execute(createTableQuery1); + database.execute(createTableQuery2); + database.execute(insertIntoTableQuery1); + database.execute(insertIntoTableQuery2); + + database.close(); + } + + @Override + protected void tearDown(TestDestinationEnv testEnv) { + new File("certificate.pem").delete(); + new File("certificate.der").delete(); + new File(KEY_STORE_FILE_PATH).delete(); + db.close(); + } + + /* Helpers */ + + private String getCertificate() throws IOException, InterruptedException { + // To enable SSL connection on the server, we need to generate self-signed certificates for the server and add them to the configuration. + // Then you need to enable SSL connection and specify on which port it will work. These changes will take effect after restart. + // The certificate for generating a user certificate has the extension *.arm. + db.execInContainer("su", "-", "db2inst1", "-c", "gsk8capicmd_64 -keydb -create -db \"server.kdb\" -pw \"" + TEST_KEY_STORE_PASS + "\" -stash"); + db.execInContainer("su", "-", "db2inst1", "-c", "gsk8capicmd_64 -cert -create -db \"server.kdb\" -pw \"" + TEST_KEY_STORE_PASS + + "\" -label \"mylabel\" -dn \"CN=testcompany\" -size 2048 -sigalg SHA256_WITH_RSA"); + db.execInContainer("su", "-", "db2inst1", "-c", "gsk8capicmd_64 -cert -extract -db \"server.kdb\" -pw \"" + TEST_KEY_STORE_PASS + + "\" -label \"mylabel\" -target \"server.arm\" -format ascii -fips"); + + db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_SVR_KEYDB /database/config/db2inst1/server.kdb"); + db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_SVR_STASH /database/config/db2inst1/server.sth"); + db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_SVR_LABEL mylabel"); + db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_SVCENAME 50000"); + db.execInContainer("su", "-", "db2inst1", "-c", "db2set -i db2inst1 DB2COMM=SSL"); + db.execInContainer("su", "-", "db2inst1", "-c", "db2stop force"); + db.execInContainer("su", "-", "db2inst1", "-c", "db2start"); + return db.execInContainer("su", "-", "db2inst1", "-c", "cat server.arm").getStdout(); + } + + private static void convertAndImportCertificate(String certificate) throws IOException, InterruptedException { + Runtime run = Runtime.getRuntime(); + try (PrintWriter out = new PrintWriter("certificate.pem")) { + out.print(certificate); + } + runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run); + runProcess( + "keytool -import -alias rds-root -keystore " + KEY_STORE_FILE_PATH + " -file certificate.der -storepass " + TEST_KEY_STORE_PASS + + " -noprompt", + run); + } + + private static void runProcess(String cmd, Runtime run) throws IOException, InterruptedException { + Process pr = run.exec(cmd); + if (!pr.waitFor(30, TimeUnit.SECONDS)) { + pr.destroy(); + throw new RuntimeException("Timeout while executing: " + cmd); + } + } + +} diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2JdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2JdbcSourceAcceptanceTest.java new file mode 100644 index 0000000000000..266494da20ea7 --- /dev/null +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2JdbcSourceAcceptanceTest.java @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.db2_strict_encrypt; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.base.Source; +import io.airbyte.integrations.source.db2.Db2Source; +import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.protocol.models.ConnectorSpecification; +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Collections; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.Db2Container; + +class Db2JdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { + + private static final String TEST_KEY_STORE_PASS = "Passw0rd"; + private static final String KEY_STORE_FILE_PATH = "clientkeystore.jks"; + + private static Set TEST_TABLES = Collections.emptySet(); + private static String certificate; + private static Db2Container db; + private JsonNode config; + + @BeforeAll + static void init() throws IOException, InterruptedException { + db = new Db2Container("ibmcom/db2:11.5.5.0").acceptLicense(); + db.start(); + + certificate = getCertificate(); + try { + convertAndImportCertificate(certificate); + } catch (IOException | InterruptedException e) { + throw new RuntimeException("Failed to import certificate into Java Keystore"); + } + + // Db2 transforms names to upper case, so we need to use upper case name to retrieve data later. + SCHEMA_NAME = "JDBC_INTEGRATION_TEST1"; + SCHEMA_NAME2 = "JDBC_INTEGRATION_TEST2"; + TEST_SCHEMAS = ImmutableSet.of(SCHEMA_NAME, SCHEMA_NAME2); + TABLE_NAME = "ID_AND_NAME"; + TABLE_NAME_WITH_SPACES = "ID AND NAME"; + TABLE_NAME_WITHOUT_PK = "ID_AND_NAME_WITHOUT_PK"; + TABLE_NAME_COMPOSITE_PK = "FULL_NAME_COMPOSITE_PK"; + TEST_TABLES = ImmutableSet + .of(TABLE_NAME, TABLE_NAME_WITHOUT_PK, TABLE_NAME_COMPOSITE_PK); + COL_ID = "ID"; + COL_NAME = "NAME"; + COL_UPDATED_AT = "UPDATED_AT"; + COL_FIRST_NAME = "FIRST_NAME"; + COL_LAST_NAME = "LAST_NAME"; + COL_LAST_NAME_WITH_SPACE = "LAST NAME"; + // In Db2 PK columns must be declared with NOT NULL statement. + COLUMN_CLAUSE_WITH_PK = "id INTEGER NOT NULL, name VARCHAR(200), updated_at DATE"; + COLUMN_CLAUSE_WITH_COMPOSITE_PK = "first_name VARCHAR(200) NOT NULL, last_name VARCHAR(200) NOT NULL, updated_at DATE"; + // There is no IF EXISTS statement for a schema in Db2. + // The schema name must be in the catalog when attempting the DROP statement; otherwise an error is + // returned. + DROP_SCHEMA_QUERY = "DROP SCHEMA %s RESTRICT"; + } + + @BeforeEach + public void setup() throws Exception { + config = Jsons.jsonNode(ImmutableMap.builder() + .put("host", db.getHost()) + .put("port", db.getFirstMappedPort()) + .put("db", db.getDatabaseName()) + .put("username", db.getUsername()) + .put("password", db.getPassword()) + .put("encryption", Jsons.jsonNode(ImmutableMap.builder() + .put("encryption_method", "encrypted_verify_certificate") + .put("ssl_certificate", certificate) + .put("key_store_password", TEST_KEY_STORE_PASS) + .build())) + .build()); + + super.setup(); + } + + @AfterEach + public void clean() throws Exception { + // In Db2 before dropping a schema, all objects that were in that schema must be dropped or moved to + // another schema. + for (final String tableName : TEST_TABLES) { + final String dropTableQuery = String + .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, tableName); + super.database.execute(connection -> connection.createStatement().execute(dropTableQuery)); + } + for (int i = 2; i < 10; i++) { + final String dropTableQuery = String + .format("DROP TABLE IF EXISTS %s.%s%s", SCHEMA_NAME, TABLE_NAME, i); + super.database.execute(connection -> connection.createStatement().execute(dropTableQuery)); + } + super.database.execute(connection -> connection.createStatement().execute(String + .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, + sourceOperations.enquoteIdentifier(connection, TABLE_NAME_WITH_SPACES)))); + super.database.execute(connection -> connection.createStatement().execute(String + .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, + sourceOperations.enquoteIdentifier(connection, TABLE_NAME_WITH_SPACES + 2)))); + super.database.execute(connection -> connection.createStatement().execute(String + .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME2, + sourceOperations.enquoteIdentifier(connection, TABLE_NAME)))); + + super.tearDown(); + } + + @AfterAll + static void cleanUp() { + new File("certificate.pem").delete(); + new File("certificate.der").delete(); + new File(KEY_STORE_FILE_PATH).delete(); + db.close(); + } + + @Override + public boolean supportsSchemas() { + return true; + } + + @Override + public JsonNode getConfig() { + return Jsons.clone(config); + } + + @Override + public String getDriverClass() { + return Db2StrictEncryptSource.DRIVER_CLASS; + } + + @Override + public AbstractJdbcSource getJdbcSource() { + return new Db2Source(); + } + + @Override + public Source getSource() { + return new Db2StrictEncryptSource(); + } + + @Test + void testSpec() throws Exception { + final ConnectorSpecification actual = source.spec(); + final ConnectorSpecification expected = + Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class); + + assertEquals(expected, actual); + } + + /* Helpers */ + + private static String getCertificate() throws IOException, InterruptedException { + db.execInContainer("su", "-", "db2inst1", "-c", "gsk8capicmd_64 -keydb -create -db \"server.kdb\" -pw \"" + TEST_KEY_STORE_PASS + "\" -stash"); + db.execInContainer("su", "-", "db2inst1", "-c", "gsk8capicmd_64 -cert -create -db \"server.kdb\" -pw \"" + TEST_KEY_STORE_PASS + + "\" -label \"mylabel\" -dn \"CN=testcompany\" -size 2048 -sigalg SHA256_WITH_RSA"); + db.execInContainer("su", "-", "db2inst1", "-c", "gsk8capicmd_64 -cert -extract -db \"server.kdb\" -pw \"" + TEST_KEY_STORE_PASS + + "\" -label \"mylabel\" -target \"server.arm\" -format ascii -fips"); + + db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_SVR_KEYDB /database/config/db2inst1/server.kdb"); + db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_SVR_STASH /database/config/db2inst1/server.sth"); + db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_SVR_LABEL mylabel"); + db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_SVCENAME 50000"); + db.execInContainer("su", "-", "db2inst1", "-c", "db2set -i db2inst1 DB2COMM=SSL"); + db.execInContainer("su", "-", "db2inst1", "-c", "db2stop force"); + db.execInContainer("su", "-", "db2inst1", "-c", "db2start"); + return db.execInContainer("su", "-", "db2inst1", "-c", "cat server.arm").getStdout(); + } + + private static void convertAndImportCertificate(String certificate) throws IOException, InterruptedException { + Runtime run = Runtime.getRuntime(); + try (PrintWriter out = new PrintWriter("certificate.pem")) { + out.print(certificate); + } + runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run); + runProcess( + "keytool -import -alias rds-root -keystore " + KEY_STORE_FILE_PATH + " -file certificate.der -storepass " + TEST_KEY_STORE_PASS + + " -noprompt", + run); + } + + private static void runProcess(String cmd, Runtime run) throws IOException, InterruptedException { + Process pr = run.exec(cmd); + if (!pr.waitFor(30, TimeUnit.SECONDS)) { + pr.destroy(); + throw new RuntimeException("Timeout while executing: " + cmd); + } + } + +} diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/resources/expected_spec.json new file mode 100644 index 0000000000000..ce5f76dad3213 --- /dev/null +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/resources/expected_spec.json @@ -0,0 +1,78 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/db2", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "IBM Db2 Source Spec", + "type": "object", + "required": ["host", "port", "db", "username", "password", "encryption"], + "additionalProperties": false, + "properties": { + "host": { + "description": "Host of the Db2.", + "type": "string", + "order": 0 + }, + "port": { + "description": "Port of the database.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 8123, + "examples": ["8123"], + "order": 1 + }, + "db": { + "description": "Name of the database.", + "type": "string", + "examples": ["default"], + "order": 2 + }, + "username": { + "description": "Username to use to access the database.", + "type": "string", + "order": 3 + }, + "password": { + "description": "Password associated with the username.", + "type": "string", + "airbyte_secret": true, + "order": 4 + }, + "encryption": { + "title": "Encryption", + "type": "object", + "description": "Encryption method to use when communicating with the database", + "order": 5, + "oneOf": [ + { + "title": "TLS Encrypted (verify certificate)", + "additionalProperties": false, + "description": "Verify and use the cert provided by the server.", + "required": ["encryption_method", "ssl_certificate"], + "properties": { + "encryption_method": { + "type": "string", + "const": "encrypted_verify_certificate", + "enum": ["encrypted_verify_certificate"], + "default": "encrypted_verify_certificate" + }, + "ssl_certificate": { + "title": "SSL PEM file", + "description": "Privacy Enhanced Mail (PEM) files are concatenated certificate containers frequently used in certificate installations", + "type": "string", + "airbyte_secret": true, + "multiline": true + }, + "key_store_password": { + "title": "Key Store Password. This field is optional. If you do not fill in this field, the password will be randomly generated.", + "description": "Key Store Password", + "type": "string", + "airbyte_secret": true + } + } + } + ] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-db2/acceptance-test-config.yml b/airbyte-integrations/connectors/source-db2/acceptance-test-config.yml new file mode 100644 index 0000000000000..0d964e5289a87 --- /dev/null +++ b/airbyte-integrations/connectors/source-db2/acceptance-test-config.yml @@ -0,0 +1,6 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-db2:dev +tests: + spec: + - spec_path: "src/main/resources/spec.json"