Destinations snowflake, bigquery, redshift: improve error reporting (#…

…34458)

airbyte-cdk/java/airbyte-cdk/README.md

@@ -166,6 +166,7 @@ MavenLocal debugging steps:
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                        |
 |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.14.2  | 2024-01-24 | [\#34458](https://github.com/airbytehq/airbyte/pull/34458) | Handle case-sensitivity in sentry error grouping                                                                                                               |
 | 0.14.1  | 2024-01-24 | [\#34468](https://github.com/airbytehq/airbyte/pull/34468) | Add wait for process to be done before ending sync in destination BaseTDTest                                                                                   |
 | 0.14.0  | 2024-01-23 | [\#34461](https://github.com/airbytehq/airbyte/pull/34461) | Revert non backward compatible signature changes from 0.13.1                                                                                                   |
 | 0.13.3  | 2024-01-23 | [\#34077](https://github.com/airbytehq/airbyte/pull/34077) | Denote if destinations fully support Destinations V2                                                                                                           |

airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java

@@ -71,22 +71,26 @@ public void uncaughtException(final Thread thread, final Throwable throwable) {
     final Optional<Throwable> deinterpolatableException = ExceptionUtils.getThrowableList(throwable).stream()
         .filter(t -> THROWABLES_TO_DEINTERPOLATE.stream().anyMatch(deinterpolatableClass -> deinterpolatableClass.isAssignableFrom(t.getClass())))
         .findFirst();
+    final boolean messageWasMangled;
     if (deinterpolatableException.isPresent()) {
+      final String originalMessage = deinterpolatableException.get().getMessage();
       mangledMessage = STRINGS_TO_DEINTERPOLATE.stream()
           // Sort the strings longest to shortest, in case any target string is a substring of another
           // e.g. "airbyte_internal" should be swapped out before "airbyte"
           .sorted(Comparator.comparing(String::length).reversed())
-          .reduce(deinterpolatableException.get().getMessage(), AirbyteExceptionHandler::deinterpolate);
+          .reduce(originalMessage, AirbyteExceptionHandler::deinterpolate);
+      messageWasMangled = !mangledMessage.equals(originalMessage);
     } else {
       mangledMessage = throwable.getMessage();
+      messageWasMangled = false;
     }
 
-    // If we did not modify the message (either not a deinterpolatable class, or we tried to
-    // deinterpolate
-    // but made no changes) then emit our default trace message
-    if (mangledMessage.equals(throwable.getMessage())) {
+    if (!messageWasMangled) {
+      // If we did not modify the message (either not a deinterpolatable class, or we tried to
+      // deinterpolate but made no changes) then emit our default trace message
       AirbyteTraceMessageUtility.emitSystemErrorTrace(throwable, logMessage);
     } else {
+      // If we did modify the message, then emit a custom trace message
       AirbyteTraceMessageUtility.emitCustomErrorTrace(throwable.getMessage(), mangledMessage);
     }
 
@@ -95,7 +99,8 @@ public void uncaughtException(final Thread thread, final Throwable throwable) {
 
   @NotNull
   private static String deinterpolate(final String message, final String targetString) {
-    final String quotedTarget = '(' + Pattern.quote(targetString) + ')';
+    // (?i) makes the pattern case-insensitive
+    final String quotedTarget = '(' + "(?i)" + Pattern.quote(targetString) + ')';
     final String targetRegex = REGEX_PREFIX + quotedTarget + REGEX_SUFFIX;
     final Pattern pattern = Pattern.compile(targetRegex);
     final Matcher matcher = pattern.matcher(message);
@@ -116,7 +121,7 @@ public static void addThrowableForDeinterpolation(final Class<? extends Throwabl
 
   public static void addStringForDeinterpolation(final String string) {
     if (string != null) {
-      STRINGS_TO_DEINTERPOLATE.add(string);
+      STRINGS_TO_DEINTERPOLATE.add(string.toLowerCase());
     }
   }
 
@@ -139,19 +144,19 @@ protected void terminate() {
   @VisibleForTesting
   static void addCommonStringsToDeinterpolate() {
     // Add some common strings to deinterpolate, regardless of what the connector is doing
-    STRINGS_TO_DEINTERPOLATE.add("airbyte");
-    STRINGS_TO_DEINTERPOLATE.add("config");
-    STRINGS_TO_DEINTERPOLATE.add("configuration");
-    STRINGS_TO_DEINTERPOLATE.add("description");
-    STRINGS_TO_DEINTERPOLATE.add("email");
-    STRINGS_TO_DEINTERPOLATE.add("id");
-    STRINGS_TO_DEINTERPOLATE.add("location");
-    STRINGS_TO_DEINTERPOLATE.add("message");
-    STRINGS_TO_DEINTERPOLATE.add("name");
-    STRINGS_TO_DEINTERPOLATE.add("state");
-    STRINGS_TO_DEINTERPOLATE.add("status");
-    STRINGS_TO_DEINTERPOLATE.add("type");
-    STRINGS_TO_DEINTERPOLATE.add("userEmail");
+    addStringForDeinterpolation("airbyte");
+    addStringForDeinterpolation("config");
+    addStringForDeinterpolation("configuration");
+    addStringForDeinterpolation("description");
+    addStringForDeinterpolation("email");
+    addStringForDeinterpolation("id");
+    addStringForDeinterpolation("location");
+    addStringForDeinterpolation("message");
+    addStringForDeinterpolation("name");
+    addStringForDeinterpolation("state");
+    addStringForDeinterpolation("status");
+    addStringForDeinterpolation("type");
+    addStringForDeinterpolation("userEmail");
   }
 
 }

airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties

@@ -1 +1 @@
-version=0.14.1
+version=0.14.2

airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandlerTest.java

@@ -64,12 +64,13 @@ void testMessageDeinterpolation() throws Exception {
     // foo and bar are added to the list explicitly
     // name and description are added implicitly by the exception handler.
     // all of them should be replaced by '?'
-    runTestWithMessage("Error happened in arst_foo_bar_zxcv (name: description)");
+    // (including FOO, which should be detected case-insensitively)
+    runTestWithMessage("Error happened in arst_FOO_bar_zxcv (name: description)");
 
     final AirbyteMessage traceMessage = findFirstTraceMessage();
     assertAll(
         () -> assertEquals(AirbyteTraceMessage.Type.ERROR, traceMessage.getTrace().getType()),
-        () -> assertEquals("Error happened in arst_foo_bar_zxcv (name: description)", traceMessage.getTrace().getError().getMessage()),
+        () -> assertEquals("Error happened in arst_FOO_bar_zxcv (name: description)", traceMessage.getTrace().getError().getMessage()),
         () -> assertEquals("Error happened in arst_?_?_zxcv (?: ?)", traceMessage.getTrace().getError().getInternalMessage()),
         () -> assertEquals(AirbyteErrorTraceMessage.FailureType.SYSTEM_ERROR, traceMessage.getTrace().getError().getFailureType()),
         () -> Assertions.assertNull(traceMessage.getTrace().getError().getStackTrace(),

airbyte-integrations/connectors/destination-bigquery/build.gradle

@@ -4,7 +4,7 @@ plugins {
 }
 
 airbyteJavaConnector {
-    cdkVersionRequired = '0.14.1'
+    cdkVersionRequired = '0.14.2'
     features = ['db-destinations', 's3-destinations', 'typing-deduping']
     useLocalCdk = false
 }

airbyte-integrations/connectors/destination-bigquery/metadata.yaml

@@ -5,7 +5,7 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: 22f6c74f-5699-40ff-833c-4a879ea40133
-  dockerImageTag: 2.4.0
+  dockerImageTag: 2.4.1
   dockerRepository: airbyte/destination-bigquery
   documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery
   githubIssueLabel: destination-bigquery

airbyte-integrations/connectors/destination-redshift/build.gradle

@@ -4,7 +4,7 @@ plugins {
 }
 
 airbyteJavaConnector {
-    cdkVersionRequired = '0.14.0'
+    cdkVersionRequired = '0.14.2'
     features = ['db-destinations', 's3-destinations', 'typing-deduping']
     useLocalCdk = false
 }

airbyte-integrations/connectors/destination-redshift/metadata.yaml

@@ -5,7 +5,7 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc
-  dockerImageTag: 2.1.0
+  dockerImageTag: 2.1.1
   dockerRepository: airbyte/destination-redshift
   documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift
   githubIssueLabel: destination-redshift

airbyte-integrations/connectors/destination-snowflake/build.gradle

@@ -4,7 +4,7 @@ plugins {
 }
 
 airbyteJavaConnector {
-    cdkVersionRequired = '0.14.0'
+    cdkVersionRequired = '0.14.2'
     features = ['db-destinations', 's3-destinations', 'typing-deduping']
     useLocalCdk = false
 }

airbyte-integrations/connectors/destination-snowflake/metadata.yaml

@@ -5,7 +5,7 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: 424892c4-daac-4491-b35d-c6688ba547ba
-  dockerImageTag: 3.5.1
+  dockerImageTag: 3.5.2
   dockerRepository: airbyte/destination-snowflake
   documentationUrl: https://docs.airbyte.com/integrations/destinations/snowflake
   githubIssueLabel: destination-snowflake

docs/integrations/destinations/bigquery.md

@@ -210,6 +210,7 @@ tutorials:
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                         |
 |:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 2.4.1   | 2024-01-24 | [34458](https://github.com/airbytehq/airbyte/pull/34458)   | Improve error reporting                                                                                                                                         |
 | 2.4.0   | 2024-01-24 | [34468](https://github.com/airbytehq/airbyte/pull/34468)   | Upgrade CDK to 0.14.0                                                                                                                                           |
 | 2.3.31  | 2024-01-22 | [\#34023](https://github.com/airbytehq/airbyte/pull/34023) | Combine DDL operations into a single execution                                                                                                                  |
 | 2.3.30  | 2024-01-12 | [\#34226](https://github.com/airbytehq/airbyte/pull/34226) | Upgrade CDK to 0.12.0; Cleanup dependencies                                                                                                                     |

docs/integrations/destinations/redshift.md

@@ -29,7 +29,7 @@ For INSERT strategy:
 2. COPY: Replicates data by first uploading data to an S3 bucket and issuing a COPY command. This is
    the recommended loading approach described by Redshift
    [best practices](https://docs.aws.amazon.com/redshift/latest/dg/c_best-practices-single-copy-command.html).
-   Requires an S3 bucket and credentials. Data is copied into S3 as multiple files with a manifest file. 
+   Requires an S3 bucket and credentials. Data is copied into S3 as multiple files with a manifest file.
 
 Airbyte automatically picks an approach depending on the given configuration - if S3 configuration
 is present, Airbyte will use the COPY strategy and vice versa.
@@ -77,8 +77,8 @@ Optional parameters:
     completes; if you want to keep them for other purposes, set `purge_staging_data` to `false`.
 
 NOTE: S3 staging does not use the SSH Tunnel option for copying data, if configured. SSH Tunnel supports the SQL
-connection only. S3 is secured through public HTTPS access only. Subsequent typing and deduping queries on final table 
-are executed over using provided SSH Tunnel configuration. 
+connection only. S3 is secured through public HTTPS access only. Subsequent typing and deduping queries on final table
+are executed over using provided SSH Tunnel configuration.
 
 ## Step 1: Set up Redshift
 
@@ -99,10 +99,10 @@ are executed over using provided SSH Tunnel configuration.
 ### Permissions in Redshift
 Airbyte writes data into two schemas, whichever schema you want your data to land in, e.g. `my_schema`
 and a "Raw Data" schema that Airbyte uses to improve ELT reliability. By default, this raw data schema
-is `airbyte_internal` but this can be overridden in the Redshift Destination's advanced settings. 
+is `airbyte_internal` but this can be overridden in the Redshift Destination's advanced settings.
 Airbyte also needs to query Redshift's
-[SVV_TABLE_INFO](https://docs.aws.amazon.com/redshift/latest/dg/r_SVV_TABLE_INFO.html) table for 
-metadata about the tables airbyte manages. 
+[SVV_TABLE_INFO](https://docs.aws.amazon.com/redshift/latest/dg/r_SVV_TABLE_INFO.html) table for
+metadata about the tables airbyte manages.
 
 To ensure the `airbyte_user` has the correction permissions to:
 - create schemas in your database
@@ -237,6 +237,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                                                                          |
 |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 2.1.1   | 2024-01-24 | [34458](https://github.com/airbytehq/airbyte/pull/34458)   | Improve error reporting                                                                                                                                                                                          |
 | 2.1.0   | 2024-01-24 | [34467](https://github.com/airbytehq/airbyte/pull/34467)   | Upgrade CDK to 0.14.0                                                                                                                                                                                            |
 | 2.0.0   | 2024-01-23 | [\#34077](https://github.com/airbytehq/airbyte/pull/34077) | Destinations V2                                                                                                                                                                                                  |
 | 0.8.0   | 2024-01-18 | [\#34236](https://github.com/airbytehq/airbyte/pull/34236) | Upgrade CDK to 0.13.0                                                                                                                                                                                            |

docs/integrations/destinations/snowflake.md

@@ -246,6 +246,7 @@ Otherwise, make sure to grant the role the required permissions in the desired n
 
 | Version         | Date       | Pull Request                                               | Subject                                                                                                                                                         |
 |:----------------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 3.5.2           | 2024-01-24 | [\#34458](https://github.com/airbytehq/airbyte/pull/34458) | Improve error reporting                                                                                                                                         |
 | 3.5.1           | 2024-01-24 | [\#34501](https://github.com/airbytehq/airbyte/pull/34501) | Internal code changes for Destinations V2                                                                                                                       |
 | 3.5.0           | 2024-01-24 | [\#34462](https://github.com/airbytehq/airbyte/pull/34462) | Upgrade CDK to 0.14.0                                                                                                                                           |
 | 3.4.22          | 2024-01-12 | [\#34227](https://github.com/airbytehq/airbyte/pull/34227) | Upgrade CDK to 0.12.0; Cleanup unused dependencies                                                                                                              |