Skip to content

Commit

Permalink
source-cockroachdb: adopt CDK 0.20.4 (#35234)
Browse files Browse the repository at this point in the history
  • Loading branch information
@postamar
postamar committed Feb 14, 2024
1 parent 9e7936e commit 3b5ecfc
Show file tree
Hide file tree
Showing 8 changed files with 75 additions and 87 deletions.
22 changes: 3 additions & 19 deletions airbyte-integrations/connectors/source-cockroachdb/build.gradle
View file
Original file line number Diff line number Diff line change
@@ -1,36 +1,20 @@
plugins {
id 'application'
id 'airbyte-java-connector'
}

airbyteJavaConnector {
cdkVersionRequired = '0.13.2'
cdkVersionRequired = '0.20.4'
features = ['db-sources']
useLocalCdk = false
}

//remove once upgrading the CDK version to 0.4.x or later
java {
compileTestJava {
options.compilerArgs.remove("-Werror")
}
compileJava {
options.compilerArgs.remove("-Werror")
}
}

airbyteJavaConnector.addCdkDependencies()

application {
mainClass = 'io.airbyte.integrations.source.cockroachdb.CockroachDbSource'
applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
}

dependencies {
implementation 'org.postgresql:postgresql:42.6.0'

implementation 'org.apache.commons:commons-lang3:3.11'
implementation libs.postgresql

testImplementation libs.testcontainers.cockroachdb
testImplementation 'org.apache.commons:commons-lang3:3.11'
testImplementation 'org.testcontainers:cockroachdb:1.19.4'
}
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-cockroachdb/metadata.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ data:
connectorSubtype: database
connectorType: source
definitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003
dockerImageTag: 0.2.1
dockerImageTag: 0.2.2
dockerRepository: airbyte/source-cockroachdb
githubIssueLabel: source-cockroachdb
icon: cockroachdb.svg
Expand Down
1 change: 1 addition & 0 deletions ...ckroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ public Set<String> getExcludedInternalNameSpaces() {
}

@Override
@SuppressWarnings("unchecked")
public Set<JdbcPrivilegeDto> getPrivilegesTableForCurrentUser(final JdbcDatabase database, final String schema) throws SQLException {
try (final Stream<JsonNode> stream = database.unsafeQuery(getPrivileges(database), sourceOperations::rowToJson)) {
return stream.map(this::getPrivilegeDto).collect(Collectors.toSet());
Expand Down
6 changes: 6 additions & 0 deletions ...achdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcDatabase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Stream;

/**
Expand Down Expand Up @@ -81,4 +82,9 @@ public Stream<JsonNode> unsafeQuery(final String sql, final String... params) th

}

@Override
public <T> T executeMetadataQuery(Function<DatabaseMetaData, T> function) throws SQLException {
return database.executeMetadataQuery(function);
}

}
28 changes: 14 additions & 14 deletions ...tion/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,27 +59,27 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc
.put(JdbcUtils.SSL_KEY, false)
.build());

try (final DSLContext dslContext = DSLContextFactory.create(
final DSLContext dslContext = DSLContextFactory.create(
config.get(JdbcUtils.USERNAME_KEY).asText(),
config.get(JdbcUtils.PASSWORD_KEY).asText(),
DatabaseDriver.POSTGRESQL.getDriverClassName(),
String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(),
config.get(JdbcUtils.HOST_KEY).asText(),
config.get(JdbcUtils.PORT_KEY).asInt(),
config.get(JdbcUtils.DATABASE_KEY).asText()),
SQLDialect.POSTGRES)) {
final Database database = new Database(dslContext);

database.query(ctx -> {
ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));");
ctx.fetch(
"INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');");
ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));");
ctx.fetch(
"INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');");
return null;
});
}
SQLDialect.POSTGRES);
final Database database = new Database(dslContext);

database.query(ctx -> {
ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));");
ctx.fetch(
"INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');");
ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));");
ctx.fetch(
"INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');");
return null;
});

}

@Override
Expand Down
1 change: 0 additions & 1 deletion ...ration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,6 @@ protected JsonNode getConfig() {

@Override
protected void tearDown(final TestDestinationEnv testEnv) {
dslContext.close();
container.close();
}

Expand Down
101 changes: 49 additions & 52 deletions ...achdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,32 +103,31 @@ void setup() throws Exception {
dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase();

final JsonNode config = getConfig(PSQL_DB, null);
try (final DSLContext dslContext = getDslContext(config)) {
final Database database = getDatabase(dslContext);
database.query(ctx -> {
ctx.fetch("CREATE DATABASE " + dbName + ";");
ctx.fetch(
"CREATE TABLE " + dbName + ".id_and_name(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));");
ctx.fetch("CREATE INDEX i1 ON " + dbName + ".id_and_name (id);");
ctx.fetch(
"INSERT INTO " + dbName
+ ".id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');");

ctx.fetch(
"CREATE TABLE " + dbName + ".id_and_name2(id NUMERIC(20, 10), name VARCHAR(200), power double precision);");
ctx.fetch(
"INSERT INTO " + dbName
+ ".id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');");

ctx.fetch(
"CREATE TABLE " + dbName
+ ".names(first_name VARCHAR(200), last_name VARCHAR(200), power double precision, PRIMARY KEY (first_name, last_name));");
ctx.fetch(
"INSERT INTO " + dbName
+ ".names (first_name, last_name, power) VALUES ('san', 'goku', 'Infinity'), ('prince', 'vegeta', 9000.1), ('piccolo', 'junior', '-Infinity');");
return null;
});
}
final DSLContext dslContext = getDslContext(config);
final Database database = getDatabase(dslContext);
database.query(ctx -> {
ctx.fetch("CREATE DATABASE " + dbName + ";");
ctx.fetch(
"CREATE TABLE " + dbName + ".id_and_name(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));");
ctx.fetch("CREATE INDEX i1 ON " + dbName + ".id_and_name (id);");
ctx.fetch(
"INSERT INTO " + dbName
+ ".id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');");

ctx.fetch(
"CREATE TABLE " + dbName + ".id_and_name2(id NUMERIC(20, 10), name VARCHAR(200), power double precision);");
ctx.fetch(
"INSERT INTO " + dbName
+ ".id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');");

ctx.fetch(
"CREATE TABLE " + dbName
+ ".names(first_name VARCHAR(200), last_name VARCHAR(200), power double precision, PRIMARY KEY (first_name, last_name));");
ctx.fetch(
"INSERT INTO " + dbName
+ ".names (first_name, last_name, power) VALUES ('san', 'goku', 'Infinity'), ('prince', 'vegeta', 9000.1), ('piccolo', 'junior', '-Infinity');");
return null;
});
}

private static Database getDatabase(final DSLContext dslContext) {
Expand Down Expand Up @@ -184,15 +183,14 @@ public void testCanReadUtf8() throws Exception {
// .withCommand("postgres -c client_encoding=sql_ascii")
db.start();
final JsonNode config = getConfig(db);
try (final DSLContext dslContext = getDslContext(config)) {
final Database database = getDatabase(dslContext);
database.query(ctx -> {
ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));");
ctx.fetch(
"INSERT INTO id_and_name (id, name) VALUES (1,E'\\u2013 someutfstring'), (2, E'\\u2215');");
return null;
});
}
final DSLContext dslContext = getDslContext(config);
final Database database = getDatabase(dslContext);
database.query(ctx -> {
ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));");
ctx.fetch(
"INSERT INTO id_and_name (id, name) VALUES (1,E'\\u2013 someutfstring'), (2, E'\\u2215');");
return null;
});

final Set<AirbyteMessage> actualMessages = MoreIterators
.toSet(new CockroachDbSource().read(config, CONFIGURED_CATALOG, null));
Expand Down Expand Up @@ -225,23 +223,22 @@ void testDiscoverWithPk() throws Exception {
@Test
void testDiscoverWithPermissions() throws Exception {
final JsonNode config = getConfig(PSQL_DB, dbName);
try (final DSLContext dslContext = getDslContext(config)) {
final Database database = getDatabase(dslContext);
database.query(ctx -> {
ctx.fetch(
"CREATE USER cock;");
ctx.fetch(
"CREATE TABLE id_and_name_perm1(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));");
ctx.fetch(
"CREATE TABLE id_and_name_perm2(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));");
ctx.fetch(
"CREATE TABLE id_and_name_perm3(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));");
ctx.fetch("grant all on database " + dbName + " to cock;");
ctx.fetch("grant all on table " + dbName + ".public.id_and_name_perm1 to cock;");
ctx.fetch("grant select on table " + dbName + ".public.id_and_name_perm2 to cock;");
return null;
});
}
final DSLContext dslContext = getDslContext(config);
final Database database = getDatabase(dslContext);
database.query(ctx -> {
ctx.fetch(
"CREATE USER cock;");
ctx.fetch(
"CREATE TABLE id_and_name_perm1(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));");
ctx.fetch(
"CREATE TABLE id_and_name_perm2(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));");
ctx.fetch(
"CREATE TABLE id_and_name_perm3(id NUMERIC(20, 10), name VARCHAR(200), power double precision, PRIMARY KEY (id));");
ctx.fetch("grant all on database " + dbName + " to cock;");
ctx.fetch("grant all on table " + dbName + ".public.id_and_name_perm1 to cock;");
ctx.fetch("grant select on table " + dbName + ".public.id_and_name_perm2 to cock;");
return null;
});

final List<String> expected = List.of("id_and_name_perm1", "id_and_name_perm2");

Expand Down
1 change: 1 addition & 0 deletions docs/integrations/sources/cockroachdb.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ Your database user should now be ready for use with Airbyte.

| Version | Date | Pull Request | Subject |
|:--------|:-----------| :--- |:------------------------------------------------------------------------------------------------------------------------------------------|
| 0.2.2 | 2024-02-13 | [35234](https://github.com/airbytehq/airbyte/pull/35234) | Adopt CDK 0.20.4 |
| 0.2.1 | 2024-01-24 | [34453](https://github.com/airbytehq/airbyte/pull/34453) | bump CDK version |
| 0.2.0 | 2023-12-18 | [33485](https://github.com/airbytehq/airbyte/pull/33485) | Removed LEGACY state |
| 0.1.22 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting |
Expand Down

0 comments on commit 3b5ecfc

Please sign in to comment.