-
Notifications
You must be signed in to change notification settings - Fork 4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Snowflake destination : Enable DAT tests (#12912)
* enable new DAT for Snowflake and fix timezone transformation * Make Snowflake in line with new DAT tests. + move method of putting text values into node to util a class + format * Rollback to DestinationAcceptanceTest as a parent class.
- Loading branch information
1 parent
abc038e
commit 28ccd06
Showing
10 changed files
with
155 additions
and
39 deletions.
There are no files selected for viewing
21 changes: 21 additions & 0 deletions
21
...java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTestUtils.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| /* | ||
| * Copyright (c) 2021 Airbyte, Inc., all rights reserved. | ||
| */ | ||
|
|
||
| package io.airbyte.integrations.standardtest.destination; | ||
|
|
||
| import com.fasterxml.jackson.databind.node.ObjectNode; | ||
| import io.airbyte.commons.json.Jsons; | ||
|
|
||
| public class DestinationAcceptanceTestUtils { | ||
|
|
||
| public static void putStringIntoJson(String stringValue, String fieldName, ObjectNode node) { | ||
| if (stringValue != null && (stringValue.startsWith("[") && stringValue.endsWith("]") | ||
| || stringValue.startsWith("{") && stringValue.endsWith("}"))) { | ||
| node.set(fieldName, Jsons.deserialize(stringValue)); | ||
| } else { | ||
| node.put(fieldName, stringValue); | ||
| } | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
75 changes: 75 additions & 0 deletions
75
...ation/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestDataComparator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| /* | ||
| * Copyright (c) 2021 Airbyte, Inc., all rights reserved. | ||
| */ | ||
|
|
||
| package io.airbyte.integrations.destination.snowflake; | ||
|
|
||
| import io.airbyte.integrations.destination.NamingConventionTransformer; | ||
| import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; | ||
| import java.time.LocalDate; | ||
| import java.time.LocalDateTime; | ||
| import java.time.ZoneOffset; | ||
| import java.time.ZonedDateTime; | ||
| import java.time.format.DateTimeFormatter; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
|
|
||
| public class SnowflakeTestDataComparator extends AdvancedTestDataComparator { | ||
|
|
||
| public static final NamingConventionTransformer NAME_TRANSFORMER = new SnowflakeSQLNameTransformer(); | ||
|
|
||
| private static final String SNOWFLAKE_DATETIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"; | ||
| private static final String SNOWFLAKE_DATE_FORMAT = "yyyy-MM-dd"; | ||
| private static final String POSTGRES_DATETIME_WITH_TZ_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"; | ||
|
|
||
| @Override | ||
| protected List<String> resolveIdentifier(final String identifier) { | ||
| final List<String> result = new ArrayList<>(); | ||
| final String resolved = NAME_TRANSFORMER.getIdentifier(identifier); | ||
| result.add(identifier); | ||
| result.add(resolved); | ||
| if (!resolved.startsWith("\"")) { | ||
| result.add(resolved.toLowerCase()); | ||
| result.add(resolved.toUpperCase()); | ||
| } | ||
| return result; | ||
| } | ||
|
|
||
| private LocalDate parseDate(String dateValue) { | ||
| if (dateValue != null) { | ||
| var format = (dateValue.matches(".+Z") ? SNOWFLAKE_DATETIME_FORMAT : SNOWFLAKE_DATE_FORMAT); | ||
| return LocalDate.parse(dateValue, DateTimeFormatter.ofPattern(format)); | ||
| } else { | ||
| return null; | ||
| } | ||
| } | ||
|
|
||
| private LocalDate parseLocalDate(String dateTimeValue) { | ||
| if (dateTimeValue != null) { | ||
| var format = (dateTimeValue.matches(".+Z") ? POSTGRES_DATETIME_WITH_TZ_FORMAT : AIRBYTE_DATETIME_FORMAT); | ||
| return LocalDate.parse(dateTimeValue, DateTimeFormatter.ofPattern(format)); | ||
| } else { | ||
| return null; | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| protected boolean compareDateTimeValues(String expectedValue, String actualValue) { | ||
| var destinationDate = parseLocalDate(actualValue); | ||
| var expectedDate = LocalDate.parse(expectedValue, DateTimeFormatter.ofPattern(AIRBYTE_DATETIME_FORMAT)); | ||
| return expectedDate.equals(destinationDate); | ||
| } | ||
|
|
||
| @Override | ||
| protected boolean compareDateValues(String expectedValue, String actualValue) { | ||
| var destinationDate = parseDate(actualValue); | ||
| var expectedDate = LocalDate.parse(expectedValue, DateTimeFormatter.ofPattern(AIRBYTE_DATE_FORMAT)); | ||
| return expectedDate.equals(destinationDate); | ||
| } | ||
|
|
||
| @Override | ||
| protected ZonedDateTime parseDestinationDateWithTz(String destinationValue) { | ||
| return ZonedDateTime.of(LocalDateTime.parse(destinationValue, DateTimeFormatter.ofPattern(POSTGRES_DATETIME_WITH_TZ_FORMAT)), ZoneOffset.UTC); | ||
| } | ||
|
|
||
| } |
20 changes: 20 additions & 0 deletions
20
...ion/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestSourceOperations.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| /* | ||
| * Copyright (c) 2021 Airbyte, Inc., all rights reserved. | ||
| */ | ||
|
|
||
| package io.airbyte.integrations.destination.snowflake; | ||
|
|
||
| import com.fasterxml.jackson.databind.node.ObjectNode; | ||
| import io.airbyte.db.jdbc.JdbcSourceOperations; | ||
| import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTestUtils; | ||
| import java.sql.ResultSet; | ||
| import java.sql.SQLException; | ||
|
|
||
| public class SnowflakeTestSourceOperations extends JdbcSourceOperations { | ||
|
|
||
| @Override | ||
| protected void putString(ObjectNode node, String columnName, ResultSet resultSet, int index) throws SQLException { | ||
| DestinationAcceptanceTestUtils.putStringIntoJson(resultSet.getString(index), columnName, node); | ||
| } | ||
|
|
||
| } |