Skip to content

Commit

Permalink
Snowflake Destination: Upgrade CDK (#34462)
Browse files Browse the repository at this point in the history 
## What
* Upgrade Snowflake to pick the reverted CDK changes to move forward.
  • Loading branch information
@gisripa
gisripa committed Jan 24, 2024
1 parent 852cb59 commit 950fea1
Show file tree
Hide file tree
Showing 6 changed files with 7 additions and 22 deletions.
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/destination-snowflake/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ plugins {
}

airbyteJavaConnector {
cdkVersionRequired = '0.12.0'
cdkVersionRequired = '0.14.0'
features = ['db-destinations', 's3-destinations', 'typing-deduping']
useLocalCdk = false
}
Expand Down
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/destination-snowflake/metadata.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ data:
connectorSubtype: database
connectorType: destination
definitionId: 424892c4-daac-4491-b35d-c6688ba547ba
dockerImageTag: 3.4.22
dockerImageTag: 3.5.0
dockerRepository: airbyte/destination-snowflake
documentationUrl: https://docs.airbyte.com/integrations/destinations/snowflake
githubIssueLabel: destination-snowflake
Expand Down
2 changes: 1 addition & 1 deletion ...va/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ private static void attemptStageOperations(final String outputSchema,
sqlOperations.attemptWriteToStage(outputSchema, stageName, database);
} finally {
// drop created tmp stage
sqlOperations.dropStageIfExists(database, stageName);
sqlOperations.dropStageIfExists(database, stageName, null);
}
}

Expand Down
13 changes: 1 addition & 12 deletions .../io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperations.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ protected String getCopyQuery(final String stageName,
}

@Override
public void dropStageIfExists(final JdbcDatabase database, final String stageName) throws Exception {
public void dropStageIfExists(final JdbcDatabase database, final String stageName, final String stagingPath) throws Exception {
try {
final String query = getDropQuery(stageName);
LOGGER.debug("Executing query: {}", query);
Expand All @@ -222,17 +222,6 @@ protected String getDropQuery(final String stageName) {
return String.format(DROP_STAGE_QUERY, stageName);
}

@Override
public void cleanUpStage(final JdbcDatabase database, final String stageName, final List<String> stagedFiles) throws Exception {
try {
final String query = getRemoveQuery(stageName);
LOGGER.debug("Executing query: {}", query);
database.execute(query);
} catch (final SQLException e) {
throw checkForKnownConfigExceptions(e).orElseThrow(() -> e);
}
}

/**
* Creates a SQL query used to remove staging files that were just staged See
* https://docs.snowflake.com/en/sql-reference/sql/remove.html for more context
Expand Down
7 changes: 1 addition & 6 deletions ...te/integrations/destination/snowflake/SnowflakeSqlOperationsThrowConfigExceptionTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@ class SnowflakeSqlOperationsThrowConfigExceptionTest {

private static Executable createStageIfNotExists;
private static Executable dropStageIfExists;
private static Executable cleanUpStage;
private static Executable copyIntoTableFromStage;

private static Executable createSchemaIfNotExists;
Expand All @@ -65,8 +64,7 @@ public static void setup() {
snowflakeSqlOperations = new SnowflakeSqlOperations();

createStageIfNotExists = () -> snowflakeStagingSqlOperations.createStageIfNotExists(dbForExecuteQuery, STAGE_NAME);
dropStageIfExists = () -> snowflakeStagingSqlOperations.dropStageIfExists(dbForExecuteQuery, STAGE_NAME);
cleanUpStage = () -> snowflakeStagingSqlOperations.cleanUpStage(dbForExecuteQuery, STAGE_NAME, FILE_PATH);
dropStageIfExists = () -> snowflakeStagingSqlOperations.dropStageIfExists(dbForExecuteQuery, STAGE_NAME, null);
copyIntoTableFromStage =
() -> snowflakeStagingSqlOperations.copyIntoTableFromStage(dbForExecuteQuery, STAGE_NAME, STAGE_PATH, FILE_PATH, TABLE_NAME, SCHEMA_NAME);

Expand All @@ -84,9 +82,6 @@ private static Stream<Arguments> testArgumentsForDbExecute() {
Arguments.of(TEST_NO_CONFIG_EXCEPTION_CATCHED, false, dropStageIfExists),
Arguments.of(TEST_PERMISSION_EXCEPTION_CATCHED, true, dropStageIfExists),
Arguments.of(TEST_IP_NOT_IN_WHITE_LIST_EXCEPTION_CATCHED, true, dropStageIfExists),
Arguments.of(TEST_NO_CONFIG_EXCEPTION_CATCHED, false, cleanUpStage),
Arguments.of(TEST_PERMISSION_EXCEPTION_CATCHED, true, cleanUpStage),
Arguments.of(TEST_IP_NOT_IN_WHITE_LIST_EXCEPTION_CATCHED, true, cleanUpStage),
Arguments.of(TEST_NO_CONFIG_EXCEPTION_CATCHED, false, copyIntoTableFromStage),
Arguments.of(TEST_PERMISSION_EXCEPTION_CATCHED, true, copyIntoTableFromStage),
Arguments.of(TEST_IP_NOT_IN_WHITE_LIST_EXCEPTION_CATCHED, true, copyIntoTableFromStage),
Expand Down
3 changes: 2 additions & 1 deletion docs/integrations/destinations/snowflake.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,7 @@ Otherwise, make sure to grant the role the required permissions in the desired n

| Version | Date | Pull Request | Subject |
|:----------------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------|
| 3.5.0 | 2024-01-24 | [34462](https://github.com/airbytehq/airbyte/pull/34462) | Upgrade CDK to 0.14.0 |
| 3.4.22 | 2024-01-12 | [\#34227](https://github.com/airbytehq/airbyte/pull/34227) | Upgrade CDK to 0.12.0; Cleanup unused dependencies |
| 3.4.21 | 2024-01-10 | [\#34083](https://github.com/airbytehq/airbyte/pull/34083) | Emit destination stats as part of the state message |
| 3.4.20 | 2024-01-05 | [\#33948](https://github.com/airbytehq/airbyte/pull/33948) | Skip retrieving initial table state when setup fails |
Expand Down Expand Up @@ -403,4 +404,4 @@ Otherwise, make sure to grant the role the required permissions in the desired n
| 0.3.13 | 2021-09-01 | [\#5784](https://github.com/airbytehq/airbyte/pull/5784) | Updated query timeout from 30 minutes to 3 hours |
| 0.3.12 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json |
| 0.3.11 | 2021-07-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer |
| 0.3.10 | 2021-07-12 | [\#4713](https://github.com/airbytehq/airbyte/pull/4713) | Tag traffic with `airbyte` label to enable optimization opportunities from Snowflake |
| 0.3.10 | 2021-07-12 | [\#4713](https://github.com/airbytehq/airbyte/pull/4713) | Tag traffic with `airbyte` label to enable optimization opportunities from Snowflake |

0 comments on commit 950fea1

Please sign in to comment.