Merge branch 'master' into ddavydov/#1470-source-s3-do-not-infer-sche…

…ma-of-all-files
airbytehq · Feb 22, 2023 · d211e74 · d211e74
2 parents c7f8289 + fbd6dbf
commit d211e74
Show file tree

Hide file tree

Showing 97 changed files with 252 additions and 404 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -6,20 +6,6 @@
 # Protocol related items
 /docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers
 
-# Airbyte Maintainers
-airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java @airbytehq/airbyte-maintainers
-airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/BaseConfigDatabaseTest.java @airbytehq/airbyte-maintainers
-airbyte-commons/src/main/ @airbytehq/airbyte-maintainers
-airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java @airbytehq/airbyte-maintainers
-airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java @airbytehq/airbyte-maintainers
-airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @airbytehq/airbyte-maintainers
-airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/ @airbytehq/airbyte-maintainers
-airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java @airbytehq/airbyte-maintainers
-airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryWriter.java @airbytehq/airbyte-maintainers
-airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @airbytehq/airbyte-maintainers
-*build.gradle @airbytehq/airbyte-maintainers
-airbyte-integrations/connectors/**/build.gradle # ignore gradle files for individual connectors.
-
 # Normalization
 /airbyte-integrations/bases/base-normalization/ @airbytehq/destinations
 

diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java
@@ -191,7 +191,7 @@ public Process create(final String jobType,
   static List<String> localDebuggingOptions(final String containerName) {
     final boolean shouldAddDebuggerOptions =
         Optional.ofNullable(System.getenv("DEBUG_CONTAINER_IMAGE")).filter(StringUtils::isNotEmpty)
-            .map(ProcessFactory.extractShortImageName(containerName)::equals).orElse(false)
+            .map(imageName -> ProcessFactory.extractShortImageName(containerName).startsWith(imageName)).orElse(false)
             && Optional.ofNullable(System.getenv("DEBUG_CONTAINER_JAVA_OPTS")).isPresent();
     if (shouldAddDebuggerOptions) {
       return List.of("-e", "JAVA_TOOL_OPTIONS=" + System.getenv("DEBUG_CONTAINER_JAVA_OPTS"), "-p5005:5005");

diff --git a/airbyte-config/init/src/main/resources/icons/exasol.svg b/airbyte-config/init/src/main/resources/icons/exasol.svg
diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -131,7 +131,7 @@
 - name: Exasol
   destinationDefinitionId: bb6071d9-6f34-4766-bec2-d1d4ed81a653
   dockerRepository: airbyte/destination-exasol
-  dockerImageTag: 0.1.0
+  dockerImageTag: 0.1.1
   documentationUrl: https://docs.airbyte.com/integrations/destinations/exasol
   releaseStage: alpha
 - name: Firebolt
@@ -432,7 +432,7 @@
 - name: Databend
   destinationDefinitionId: 302e4d8e-08d3-4098-acd4-ac67ca365b88
   dockerRepository: airbyte/destination-databend
-  dockerImageTag: 0.1.0
+  dockerImageTag: 0.1.1
   icon: databend.svg
   documentationUrl: https://docs.airbyte.com/integrations/destinations/databend
   releaseStage: alpha

diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml
@@ -2238,7 +2238,7 @@
     - "overwrite"
     - "append"
     supportsNamespaces: true
-- dockerImage: "airbyte/destination-exasol:0.1.0"
+- dockerImage: "airbyte/destination-exasol:0.1.1"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/destinations/exasol"
     connectionSpecification:
@@ -7254,7 +7254,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-databend:0.1.0"
+- dockerImage: "airbyte/destination-databend:0.1.1"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/destinations/databend"
     connectionSpecification:
@@ -7272,14 +7272,6 @@
           description: "Hostname of the database."
           type: "string"
           order: 0
-        protocol:
-          title: "Protocol"
-          description: "Protocol of the host."
-          type: "string"
-          examples:
-          - "https"
-          default: "https"
-          order: 1
         port:
           title: "Port"
           description: "Port of the database."

diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -1385,7 +1385,7 @@
 - name: Postgres
   sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
   dockerRepository: airbyte/source-postgres
-  dockerImageTag: 1.0.45
+  dockerImageTag: 1.0.46
   documentationUrl: https://docs.airbyte.com/integrations/sources/postgres
   icon: postgresql.svg
   sourceType: database
@@ -1469,7 +1469,7 @@
 - name: Recharge
   sourceDefinitionId: 45d2e135-2ede-49e1-939f-3e3ec357a65e
   dockerRepository: airbyte/source-recharge
-  dockerImageTag: 0.2.5
+  dockerImageTag: 0.2.6
   documentationUrl: https://docs.airbyte.com/integrations/sources/recharge
   icon: recharge.svg
   sourceType: api
@@ -1649,7 +1649,7 @@
 - name: Slack
   sourceDefinitionId: c2281cee-86f9-4a86-bb48-d23286b4c7bd
   dockerRepository: airbyte/source-slack
-  dockerImageTag: 0.1.22
+  dockerImageTag: 0.1.23
   documentationUrl: https://docs.airbyte.com/integrations/sources/slack
   icon: slack.svg
   sourceType: api
@@ -1693,7 +1693,7 @@
 - name: Snowflake
   sourceDefinitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2
   dockerRepository: airbyte/source-snowflake
-  dockerImageTag: 0.1.29
+  dockerImageTag: 0.1.30
   documentationUrl: https://docs.airbyte.com/integrations/sources/snowflake
   icon: snowflake.svg
   sourceType: database
@@ -1800,7 +1800,7 @@
 - name: TikTok Marketing
   sourceDefinitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35
   dockerRepository: airbyte/source-tiktok-marketing
-  dockerImageTag: 2.0.1
+  dockerImageTag: 2.0.2
   documentationUrl: https://docs.airbyte.com/integrations/sources/tiktok-marketing
   icon: tiktok.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -11617,7 +11617,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-postgres:1.0.45"
+- dockerImage: "airbyte/source-postgres:1.0.46"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
     connectionSpecification:
@@ -12338,7 +12338,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-recharge:0.2.5"
+- dockerImage: "airbyte/source-recharge:0.2.6"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/recharge"
     connectionSpecification:
@@ -13634,7 +13634,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-slack:0.1.22"
+- dockerImage: "airbyte/source-slack:0.1.23"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/slack"
     connectionSpecification:
@@ -13950,7 +13950,7 @@
         - - "client_secret"
         oauthFlowOutputParameters:
         - - "refresh_token"
-- dockerImage: "airbyte/source-snowflake:0.1.29"
+- dockerImage: "airbyte/source-snowflake:0.1.30"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/snowflake"
     connectionSpecification:
@@ -14959,7 +14959,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-tiktok-marketing:2.0.1"
+- dockerImage: "airbyte/source-tiktok-marketing:2.0.2"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/tiktok-marketing"
     changelogUrl: "https://docs.airbyte.com/integrations/sources/tiktok-marketing"

diff --git a/airbyte-config/specs/build.gradle b/airbyte-config/specs/build.gradle
@@ -21,16 +21,6 @@ task generateSeedConnectorSpecs(type: JavaExec, dependsOn: compileJava) {
     args new File(project(":airbyte-config:init").projectDir, '/src/main/resources/seed')
 }
 
-task downloadCombinedConnectorCatalog(type: JavaExec, dependsOn: compileJava) {
-    classpath = sourceSets.main.runtimeClasspath
-    mainClass = 'io.airbyte.config.specs.CombinedConnectorCatalogDownloader'
-    args project(":airbyte-config:init").projectDir
-}
-
-project(":airbyte-config:init").tasks.processResources.dependsOn(downloadCombinedConnectorCatalog)
-project(":airbyte-config:init").tasks.processTestResources.dependsOn(downloadCombinedConnectorCatalog)
-project(":airbyte-config:init").tasks.test.dependsOn(downloadCombinedConnectorCatalog)
-
 
 task generateConnectorSpecsMask(type: JavaExec, dependsOn: generateSeedConnectorSpecs) {
     classpath = sourceSets.main.runtimeClasspath
@@ -56,3 +46,5 @@ task generateOssConnectorCatalog(type: JavaExec, dependsOn: generateSeedConnecto
 }
 
 Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)
+
+project(":airbyte-config:init").tasks.processResources.dependsOn(generateOssConnectorCatalog)
diff --git a/...onfig/specs/src/main/java/io/airbyte/config/specs/CombinedConnectorCatalogDownloader.java b/...onfig/specs/src/main/java/io/airbyte/config/specs/CombinedConnectorCatalogDownloader.java
diff --git a/...te-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java b/...te-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java
@@ -187,7 +187,7 @@ protected void setDate(final PreparedStatement preparedStatement, final int para
     }
   }
 
-  private void setDateAsTimestamp(PreparedStatement preparedStatement, int parameterIndex, String value) throws SQLException {
+  private void setDateAsTimestamp(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException {
     try {
       final Timestamp from = Timestamp.from(DataTypeUtils.getDateFormat().parse(value).toInstant());
       preparedStatement.setDate(parameterIndex, new Date(from.getTime()));
@@ -217,7 +217,7 @@ protected void setInteger(final PreparedStatement preparedStatement, final int p
   }
 
   protected void setBigInteger(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException {
-    preparedStatement.setLong(parameterIndex, Long.parseLong(value));
+    preparedStatement.setLong(parameterIndex, new BigDecimal(value).toBigInteger().longValue());
   }
 
   protected void setDouble(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException {

diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java
@@ -38,6 +38,8 @@
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
 import org.testcontainers.containers.PostgreSQLContainer;
 import org.testcontainers.utility.MountableFile;
 
@@ -398,4 +400,27 @@ private ObjectNode expectedValues() {
     return expected;
   }
 
+  @ParameterizedTest
+  @CsvSource({"'3E+1', 30",
+    "'30', 30",
+    "'999000000000', 999000000000",
+    "'999E+9', 999000000000",
+    "'1.79E+3', 1790"})
+  void testSetStatementSpecialValues(final String colValue, final long value) throws SQLException {
+    try (final Connection connection = dataSource.getConnection()) {
+      createTableWithAllTypes(connection);
+
+      final PreparedStatement ps = connection.prepareStatement("INSERT INTO data(bigint) VALUES(?);");
+
+      // insert the bit here to stay consistent even though setStatementField does not support it yet.
+      sourceOperations.setCursorField(ps, 1, JDBCType.BIGINT, colValue);
+      ps.execute();
+
+      assertExpectedOutputValues(connection,
+          ((ObjectNode) Jsons.jsonNode(Collections.emptyMap()))
+              .put("bigint", (long) value));
+      assertExpectedOutputTypes(connection);
+    }
+  }
+
 }
diff --git a/airbyte-integrations/bases/base-java/src/main/resources/AirbyteLogMessageTemplate.json b/airbyte-integrations/bases/base-java/src/main/resources/AirbyteLogMessageTemplate.json
@@ -6,7 +6,8 @@
       "field": "name"
     },
     "message": {
-      "$resolver": "message",
+      "$resolver": "pattern",
+      "pattern": "%level %C{1.}(%M):%L %m",
       "stringified": true
     },
     "stack_trace": {