Skip to content

Commit

Permalink
allow nullable columns to be used as cursor (#17131)
Browse files Browse the repository at this point in the history 
* allow nullable columns to be used as cursor

* add test

* fix test

* bump version

* Revert "bump version"

This reverts commit a3cf9af.

* bump version

* auto-bump connector version [ci skip]

Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
  • Loading branch information
@subodh1810 @octavia-squidington-iii
subodh1810 and octavia-squidington-iii committed Sep 26, 2022
1 parent 4ca32a4 commit cb7076b
Show file tree
Hide file tree
Showing 8 changed files with 15 additions and 9 deletions.
2 changes: 1 addition & 1 deletion airbyte-config/init/src/main/resources/seed/source_definitions.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -813,7 +813,7 @@
- name: Postgres
sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
dockerRepository: airbyte/source-postgres
dockerImageTag: 1.0.10
dockerImageTag: 1.0.11
documentationUrl: https://docs.airbyte.io/integrations/sources/postgres
icon: postgresql.svg
sourceType: database
Expand Down
2 changes: 1 addition & 1 deletion airbyte-config/init/src/main/resources/seed/source_specs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8239,7 +8239,7 @@
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
- dockerImage: "airbyte/source-postgres:1.0.10"
- dockerImage: "airbyte/source-postgres:1.0.11"
spec:
documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
connectionSpecification:
Expand Down
1 change: 0 additions & 1 deletion ...ors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,6 @@ protected List<TableInfo<CommonField<Datatype>>> discoverInternal(final JdbcData
private List<String> extractCursorFields(final List<JsonNode> fields) {
return fields.stream()
.filter(field -> isCursorType(getFieldType(field)))
.filter(field -> "NO".equals(field.get(INTERNAL_IS_NULLABLE).asText()))
.map(field -> field.get(INTERNAL_COLUMN_NAME).asText())
.collect(Collectors.toList());
}
Expand Down
5 changes: 3 additions & 2 deletions .../testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -355,8 +355,9 @@ protected void testDiscoverWithNullableCursorFields() throws Exception {
actual.getStreams().stream().filter(s -> s.getName().equalsIgnoreCase(TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE)).findFirst().orElse(null);
assertNotNull(stream);
assertEquals(TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE.toLowerCase(), stream.getName().toLowerCase());
assertEquals(1, stream.getSupportedSyncModes().size());
assertEquals(SyncMode.FULL_REFRESH, stream.getSupportedSyncModes().get(0));
assertEquals(2, stream.getSupportedSyncModes().size());
assertTrue(stream.getSupportedSyncModes().contains(SyncMode.FULL_REFRESH));
assertTrue(stream.getSupportedSyncModes().contains(SyncMode.INCREMENTAL));
}

protected AirbyteCatalog filterOutOtherSchemas(final AirbyteCatalog catalog) {
Expand Down
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=1.0.10
LABEL io.airbyte.version=1.0.11
LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-postgres/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=1.0.10
LABEL io.airbyte.version=1.0.11
LABEL io.airbyte.name=airbyte/source-postgres
5 changes: 5 additions & 0 deletions ...ce-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -432,6 +432,11 @@ void testDiscoverDifferentGrantAvailability() throws Exception {
}

final AirbyteCatalog actual = new PostgresSource().discover(getConfig(db, "new_test_user", "new_pass"));
actual.getStreams().stream().forEach(airbyteStream -> {
assertEquals(2, airbyteStream.getSupportedSyncModes().size());
assertTrue(airbyteStream.getSupportedSyncModes().contains(SyncMode.FULL_REFRESH));
assertTrue(airbyteStream.getSupportedSyncModes().contains(SyncMode.INCREMENTAL));
});
final Set<String> tableNames = actual.getStreams().stream().map(stream -> stream.getName()).collect(Collectors.toSet());
final Set<String> expectedVisibleNames = Sets.newHashSet(
"table_granted_by_role",
Expand Down
5 changes: 3 additions & 2 deletions docs/integrations/sources/postgres.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,7 @@ Possible solutions include:

| Version | Date | Pull Request | Subject |
|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| 1.0.11 | 2022-09-26 | [17131](https://github.com/airbytehq/airbyte/pull/17131) | Allow nullable columns to be used as cursor |
| 1.0.10 | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage |
| 1.0.9 | 2022-09-13 | [16657](https://github.com/airbytehq/airbyte/pull/16657) | Improve CDC record queueing performance |
| 1.0.8 | 2022-09-08 | [16202](https://github.com/airbytehq/airbyte/pull/16202) | Adds error messaging factory to UI |
Expand All @@ -402,13 +403,13 @@ Possible solutions include:
| 0.4.43 | 2022-08-03 | [15226](https://github.com/airbytehq/airbyte/pull/15226) | Make connectionTimeoutMs configurable through JDBC url parameters |
| 0.4.42 | 2022-08-03 | [15273](https://github.com/airbytehq/airbyte/pull/15273) | Fix a bug in `0.4.36` and correctly parse the CDC initial record waiting time |
| 0.4.41 | 2022-08-03 | [15077](https://github.com/airbytehq/airbyte/pull/15077) | Sync data from beginning if the LSN is no longer valid in CDC |
| | 2022-08-03 | [14903](https://github.com/airbytehq/airbyte/pull/14903) | Emit state messages more frequently (⛔ this version has a bug; use `1.0.1` instead) |
| | 2022-08-03 | [14903](https://github.com/airbytehq/airbyte/pull/14903) | Emit state messages more frequently (⛔ this version has a bug; use `1.0.1` instead) |
| 0.4.40 | 2022-08-03 | [15187](https://github.com/airbytehq/airbyte/pull/15187) | Add support for BCE dates/timestamps |
| | 2022-08-03 | [14534](https://github.com/airbytehq/airbyte/pull/14534) | Align regular and CDC integration tests and data mappers |
| 0.4.39 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings |
| 0.4.38 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. |
| 0.4.37 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected |
| 0.4.36 | 2022-07-21 | [14451](https://github.com/airbytehq/airbyte/pull/14451) | Make initial CDC waiting time configurable (⛔ this version has a bug and will not work; use `0.4.42` instead) |
| 0.4.36 | 2022-07-21 | [14451](https://github.com/airbytehq/airbyte/pull/14451) | Make initial CDC waiting time configurable (⛔ this version has a bug and will not work; use `0.4.42` instead) |
| 0.4.35 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors |
| 0.4.34 | 2022-07-17 | [13840](https://github.com/airbytehq/airbyte/pull/13840) | Added the ability to connect using different SSL modes and SSL certificates. |
| 0.4.33 | 2022-07-14 | [14586](https://github.com/airbytehq/airbyte/pull/14586) | Validate source JDBC url parameters |
Expand Down

0 comments on commit cb7076b

Please sign in to comment.