From 5cb441ef1fc27d99bcecfafef82dd60ce9d500fc Mon Sep 17 00:00:00 2001 From: Marius Posta Date: Thu, 1 Feb 2024 11:49:46 -0500 Subject: [PATCH] cdk cleanup --- airbyte-cdk/java/airbyte-cdk/README.md | 1 + .../acceptance-test-harness/build.gradle | 41 - ...VersionedAirbyteMessageBufferedWriter.java | 38 - ...edAirbyteMessageBufferedWriterFactory.java | 49 - .../java/airbyte-cdk/airbyte-api/readme.md | 11 - .../airbyte-commons-cli/build.gradle | 7 - .../airbyte-cdk/airbyte-commons-cli/readme.md | 3 - .../airbyte-commons-protocol/build.gradle | 10 - .../protocol/AirbyteMessageMigrator.java | 78 - .../protocol/AirbyteMessageSerDeProvider.java | 95 - .../AirbyteMessageVersionedMigrator.java | 37 - ...rbyteProtocolVersionedMigratorFactory.java | 35 - .../ConfiguredAirbyteCatalogMigrator.java | 64 - .../protocol/VersionedProtocolSerializer.java | 32 - .../migrations/AirbyteMessageMigration.java | 36 - .../ConfiguredAirbyteCatalogMigration.java | 25 - .../protocol/migrations/Migration.java | 21 - .../migrations/MigrationContainer.java | 103 -- .../migrations/util/RecordMigrations.java | 271 --- .../migrations/util/SchemaMigrations.java | 143 -- .../v1/AirbyteMessageMigrationV1.java | 178 -- .../v1/CatalogMigrationV1Helper.java | 219 --- .../ConfiguredAirbyteCatalogMigrationV1.java | 54 - .../migrations/v1/SchemaMigrationV1.java | 306 --- .../serde/AirbyteMessageDeserializer.java | 16 - .../AirbyteMessageGenericDeserializer.java | 28 - .../AirbyteMessageGenericSerializer.java | 23 - .../serde/AirbyteMessageSerializer.java | 15 - .../serde/AirbyteMessageV0Deserializer.java | 16 - .../serde/AirbyteMessageV0Serializer.java | 16 - .../serde/AirbyteMessageV1Deserializer.java | 16 - .../serde/AirbyteMessageV1Serializer.java | 16 - .../protocol/AirbyteMessageMigratorTest.java | 139 -- .../AirbyteMessageSerDeProviderTest.java | 86 - .../v1/AirbyteMessageMigrationV1Test.java | 1633 ----------------- ...nfiguredAirbyteCatalogMigrationV1Test.java | 108 -- .../serde/AirbyteMessageV0SerDeTest.java | 37 - .../serde/AirbyteMessageV1SerDeTest.java | 37 - .../src/test/resources/WellKnownTypes.json | 86 - .../java/airbyte-cdk/airbyte-commons/LICENSE | 21 - .../airbyte-cdk/airbyte-commons/build.gradle | 19 - .../airbyte-cdk/airbyte-commons/readme.md | 11 - .../airbyte-json-validation/LICENSE | 21 - .../airbyte-json-validation/build.gradle | 10 - .../airbyte-json-validation/readme.md | 7 - .../build.gradle | 7 + .../copy/azure/AzureBlobStorageConfig.java | 0 .../azure/AzureBlobStorageStreamCopier.java | 0 .../AzureBlobStorageStreamCopierFactory.java | 0 .../airbyte-cdk/config-models-oss/README.md | 20 - .../config-models-oss/build.gradle | 36 - .../java/airbyte-cdk/core/build.gradle | 105 +- ...bstractJdbcCompatibleSourceOperations.java | 4 +- .../cdk/integrations/JdbcConnector.java | 9 +- .../base/AirbyteExceptionHandler.java | 2 +- .../cdk/integrations/base/ssh/SshHelpers.java | 29 - .../cdk/integrations/base/ssh/SshTunnel.java | 2 +- .../destination/jdbc/SqlOperations.java | 2 +- .../jdbc/StagingFilenameGenerator.java | 0 .../constants/GlobalDataSizeConstants.java | 0 .../destination/jdbc/copy/StreamCopier.java | 0 .../jdbc/copy/StreamCopierFactory.java | 0 .../staging/StagingOperations.java | 4 +- .../src/main/resources/version.properties | 2 +- .../check/impl/CommonDatabaseCheckTest.java | 1 - .../BufferedStreamConsumerTest.java | 2 +- .../AsyncStreamConsumerTest.java | 2 +- .../toys_database/pre_migration_schema.txt | 23 - .../test/resources/toys_database/schema.sql | 6 - .../resources/toys_database/schema_dump.txt | 24 - .../base/ssh/SshBastionContainer.java | 36 +- .../integrations/util/HostPortResolver.java | 7 - .../cdk/testutils/ContainerFactory.java | 0 .../testutils/DatabaseConnectionHelper.java | 0 .../airbyte/cdk/testutils/NonContainer.java | 0 .../airbyte/cdk/testutils/TestDatabase.java | 1 - .../datastore-bigquery/build.gradle | 7 + .../cdk/db/bigquery/BigQueryDatabase.java | 0 .../cdk/db/bigquery/BigQueryResultSet.java | 0 .../db/bigquery/BigQuerySourceOperations.java | 0 .../TempBigQueryJoolDatabaseImpl.java | 0 .../airbyte-cdk/datastore-mongo/build.gradle | 16 + .../airbyte/cdk/db/mongodb/MongoDatabase.java | 150 ++ .../db/mongodb/MongoDatabaseException.java | 15 + .../io/airbyte/cdk/db/mongodb/MongoUtils.java | 429 +++++ .../io/airbyte/cdk/db/mongodb/TreeNode.java | 56 + .../cdk/db/mongodb/MongoUtilsTest.java | 42 + .../datastore-postgres/build.gradle | 9 + .../main/java/io/airbyte/cdk/db/PgLsn.java | 84 + .../java/io/airbyte/cdk/db/PostgresUtils.java | 26 + .../util/PostgresSslConnectionUtils.java | 0 .../java/io/airbyte/cdk/db/PgLsnTest.java | 0 .../io/airbyte/cdk/db/PostgresUtilsTest.java | 0 .../testutils/PostgreSQLContainerHelper.java | 0 .../airbyte-cdk/db-destinations/build.gradle | 46 +- .../destination/jdbc/WriteConfig.java | 11 +- .../staging/SerialStagingConsumerFactory.java | 5 +- .../DestinationAcceptanceTest.java | 3 +- .../java/airbyte-cdk/db-sources/build.gradle | 117 +- .../db-sources/src/main/resources/spec.json | 32 - .../resources/dummy_config.json | 4 - .../resources/expected_spec.json | 33 - .../build.gradle | 84 +- .../airbyte/api/client/AirbyteApiClient.java | 0 .../io/airbyte/api/client/PatchedLogsApi.java | 0 .../java/io/airbyte/commons/cli/Clis.java | 0 .../concurrency/CompletableFutures.java | 0 .../commons/concurrency/VoidCallable.java | 0 .../commons/concurrency/WaitingUtils.java | 0 .../constants/AirbyteSecretConstants.java | 0 .../java/io/airbyte/commons/enums/Enums.java | 0 .../exceptions/ConfigErrorException.java | 0 .../exceptions/ConnectionErrorException.java | 0 .../exceptions/SQLRuntimeException.java | 0 .../features/EnvVariableFeatureFlags.java | 0 .../commons/features/FeatureFlagHelper.java | 0 .../commons/features/FeatureFlags.java | 0 .../commons/features/FeatureFlagsWrapper.java | 0 .../commons/functional/CheckedBiConsumer.java | 0 .../commons/functional/CheckedBiFunction.java | 0 .../commons/functional/CheckedConsumer.java | 0 .../commons/functional/CheckedFunction.java | 0 .../commons/functional/CheckedSupplier.java | 0 .../main/java/io/airbyte/commons/io/IOs.java | 0 .../io/airbyte/commons/io/LineGobbler.java | 0 .../airbyte/commons/jackson/MoreMappers.java | 0 .../io/airbyte/commons/json/JsonPaths.java | 0 .../io/airbyte/commons/json/JsonSchemas.java | 0 .../java/io/airbyte/commons/json/Jsons.java | 0 .../commons/lang/CloseableConsumer.java | 0 .../airbyte/commons/lang/CloseableQueue.java | 0 .../commons/lang/CloseableShutdownHook.java | 0 .../io/airbyte/commons/lang/Exceptions.java | 0 .../io/airbyte/commons/lang/MoreBooleans.java | 0 .../commons/logging/LoggingHelper.java | 0 .../io/airbyte/commons/logging/MdcScope.java | 2 +- .../java/io/airbyte/commons/map/MoreMaps.java | 0 .../protocol/DefaultProtocolSerializer.java | 0 .../commons/protocol/ProtocolSerializer.java | 0 .../commons/resources/MoreResources.java | 0 .../stream/AirbyteStreamStatusHolder.java | 0 .../commons/stream/AirbyteStreamUtils.java | 0 .../airbyte/commons/stream/MoreStreams.java | 0 .../commons/stream/StreamStatusUtils.java | 0 .../io/airbyte/commons/string/Strings.java | 0 .../java/io/airbyte/commons/text/Names.java | 0 .../java/io/airbyte/commons/text/Sqls.java | 0 .../commons/util/AirbyteStreamAware.java | 0 .../commons/util/AutoCloseableIterator.java | 0 .../commons/util/AutoCloseableIterators.java | 0 .../commons/util/CompositeIterator.java | 0 .../util/DefaultAutoCloseableIterator.java | 0 .../util/LazyAutoCloseableIterator.java | 0 .../airbyte/commons/util/MoreIterators.java | 0 .../version/AirbyteProtocolVersion.java | 0 .../commons/version/AirbyteVersion.java | 0 .../io/airbyte/commons/version/Version.java | 0 .../commons/version/VersionDeserializer.java | 0 .../commons/version/VersionSerializer.java | 0 .../java/io/airbyte/commons/yaml/Yamls.java | 0 .../io/airbyte/configoss/AirbyteConfig.java | 0 .../configoss/AirbyteConfigValidator.java | 0 .../configoss/CatalogDefinitionsConfig.java | 0 .../io/airbyte/configoss/ConfigSchema.java | 0 .../java/io/airbyte/configoss/Configs.java | 0 .../java/io/airbyte/configoss/EnvConfigs.java | 0 .../configoss/helpers/StateMessageHelper.java | 0 .../YamlListToStandardDefinitions.java | 0 .../json/AbstractSchemaValidator.java | 0 .../json/ConfigSchemaValidator.java | 0 .../validation/json/JsonSchemaValidator.java | 0 .../json/JsonValidationException.java | 0 .../src/main/openapi/config.yaml | 0 .../src/main/resources/log4j2-test.xml | 0 .../ActorDefinitionResourceRequirements.yaml | 0 .../src/main/resources/types/ActorType.yaml | 0 .../main/resources/types/AllowedHosts.yaml | 0 .../types/CombinedConnectorCatalog.yaml | 0 .../resources/types/ConnectorJobOutput.yaml | 0 .../src/main/resources/types/DataType.yaml | 0 .../types/DestinationConnection.yaml | 0 .../types/DestinationOAuthParameter.yaml | 0 .../resources/types/DestinationSyncMode.yaml | 0 .../main/resources/types/DockerImageSpec.yaml | 0 .../main/resources/types/FailureReason.yaml | 0 .../resources/types/JobGetSpecConfig.yaml | 0 .../main/resources/types/JobSyncConfig.yaml | 0 .../src/main/resources/types/JobType.yaml | 0 .../types/NamespaceDefinitionType.yaml | 0 ...malizationDestinationDefinitionConfig.yaml | 0 .../resources/types/NotificationType.yaml | 0 .../src/main/resources/types/OperatorDbt.yaml | 0 .../types/OperatorNormalization.yaml | 0 .../main/resources/types/OperatorType.yaml | 0 .../main/resources/types/OperatorWebhook.yaml | 0 .../resources/types/ReplicationStatus.yaml | 0 .../resources/types/ResourceRequirements.yaml | 0 .../resources/types/SourceConnection.yaml | 0 .../resources/types/SourceOAuthParameter.yaml | 0 .../types/StandardCheckConnectionInput.yaml | 0 .../types/StandardCheckConnectionOutput.yaml | 0 .../types/StandardDestinationDefinition.yaml | 0 .../types/StandardDiscoverCatalogInput.yaml | 0 .../types/StandardSourceDefinition.yaml | 0 .../resources/types/StandardSyncInput.yaml | 0 .../types/StandardSyncOperation.yaml | 0 .../src/main/resources/types/State.yaml | 0 .../src/main/resources/types/StateType.yaml | 0 .../main/resources/types/StateWrapper.yaml | 0 .../resources/types/SuggestedStreams.yaml | 0 .../src/main/resources/types/SyncMode.yaml | 0 .../src/main/resources/types/SyncStats.yaml | 0 .../types/WebhookOperationConfigs.yaml | 0 .../types/WebhookOperationSummary.yaml | 0 .../types/WorkerDestinationConfig.yaml | 0 .../resources/types/WorkerSourceConfig.yaml | 0 .../api/client/AirbyteApiClientTest.java | 0 .../java/io/airbyte/commons/cli/ClisTest.java | 0 .../commons/concurrency/WaitingUtilsTest.java | 0 .../io/airbyte/commons/enums/EnumsTest.java | 0 .../features/FeatureFlagHelperTest.java | 0 .../java/io/airbyte/commons/io/IOsTest.java | 0 .../airbyte/commons/io/LineGobblerTest.java | 0 .../airbyte/commons/json/JsonPathsTest.java | 0 .../airbyte/commons/json/JsonSchemasTest.java | 0 .../io/airbyte/commons/json/JsonsTest.java | 0 .../lang/CloseableShutdownHookTest.java | 0 .../airbyte/commons/lang/ExceptionsTest.java | 0 .../commons/lang/MoreBooleansTest.java | 0 .../airbyte/commons/logging/MdcScopeTest.java | 0 .../io/airbyte/commons/map/MoreMapsTest.java | 0 .../commons/resources/MoreResourcesTest.java | 0 .../stream/AirbyteStreamStatusHolderTest.java | 0 .../commons/stream/StreamStatusUtilsTest.java | 0 .../airbyte/commons/string/StringsTest.java | 0 .../io/airbyte/commons/text/NamesTest.java | 0 .../io/airbyte/commons/text/SqlsTest.java | 0 .../util/AutoCloseableIteratorsTest.java | 0 .../commons/util/CompositeIteratorTest.java | 0 .../DefaultAutoCloseableIteratorTest.java | 0 .../util/LazyAutoCloseableIteratorTest.java | 0 .../commons/version/AirbyteVersionTest.java | 0 .../airbyte/commons/version/VersionTest.java | 0 .../io/airbyte/commons/yaml/YamlsTest.java | 0 .../airbyte/configoss/ConfigSchemaTest.java | 0 .../airbyte/configoss/DataTypeEnumTest.java | 0 .../YamlListToStandardDefinitionsTest.java | 0 .../json/JsonSchemaValidatorTest.java | 0 .../airbyte/workers/TestHarnessUtilsTest.java | 0 .../helper/CatalogClientConvertersTest.java | 0 .../helper/ConnectorConfigUpdaterTest.java | 0 .../workers/helper/FailureHelperTest.java | 0 .../DefaultAirbyteStreamFactoryTest.java | 0 .../json_schemas/composite_json_schema.json | 0 .../json_schemas/json_with_all_types.json | 0 .../json_with_array_type_fields.json | 0 .../json_with_array_type_fields_no_items.json | 0 ...ith_array_type_fields_with_composites.json | 0 .../org.mockito.plugins.MockMaker | 0 .../src/test/resources/resource_test | 0 .../src/test/resources/resource_test_a | 0 .../src/test/resources/subdir/resource_test_a | 0 .../test/resources/subdir/resource_test_sub | 0 .../test/resources/subdir/resource_test_sub_2 | 0 .../version-detection/logs-with-version.jsonl | 0 .../logs-without-spec-message.jsonl | 0 .../logs-without-version.jsonl | 0 .../java/io/airbyte/workers/TestHarness.java | 0 .../io/airbyte/workers/TestHarnessUtils.java | 0 .../io/airbyte/workers/WorkerConstants.java | 0 .../exception/TestHarnessException.java | 0 .../general/CheckConnectionTestHarness.java | 0 .../general/DbtTransformationRunner.java | 0 .../DefaultCheckConnectionTestHarness.java | 0 .../DefaultDiscoverCatalogTestHarness.java | 0 .../general/DefaultGetSpecTestHarness.java | 0 .../general/DiscoverCatalogTestHarness.java | 0 .../workers/general/GetSpecTestHarness.java | 0 .../helper/CatalogClientConverters.java | 0 .../helper/ConnectorConfigUpdater.java | 0 .../workers/helper/EntrypointEnvChecker.java | 0 .../airbyte/workers/helper/FailureHelper.java | 0 .../workers/internal/AirbyteDestination.java | 0 .../AirbyteMessageBufferedWriter.java | 0 .../AirbyteMessageBufferedWriterFactory.java | 0 .../internal/AirbyteProtocolPredicate.java | 0 .../workers/internal/AirbyteSource.java | 0 .../internal/AirbyteStreamFactory.java | 0 .../internal/DefaultAirbyteDestination.java | 0 .../DefaultAirbyteMessageBufferedWriter.java | 0 ...ltAirbyteMessageBufferedWriterFactory.java | 0 .../internal/DefaultAirbyteSource.java | 0 .../internal/DefaultAirbyteStreamFactory.java | 4 +- .../workers/internal/HeartbeatMonitor.java | 0 .../DefaultNormalizationRunner.java | 0 .../NormalizationAirbyteStreamFactory.java | 0 .../normalization/NormalizationRunner.java | 0 .../process/AirbyteIntegrationLauncher.java | 0 .../workers/process/DockerProcessFactory.java | 0 .../workers/process/IntegrationLauncher.java | 0 .../io/airbyte/workers/process/Metadata.java | 0 .../workers/process/ProcessFactory.java | 0 .../test_utils/AirbyteMessageUtils.java | 0 .../workers/test_utils/TestConfigHelpers.java | 0 .../dbt_transformation_entrypoint.sh | 0 .../resources/entrypoints/sync/check.sh | 0 .../resources/entrypoints/sync/init.sh | 0 .../resources/entrypoints/sync/main.sh | 0 .../testFixtures}/resources/image_exists.sh | 0 .../testFixtures}/resources/sshtunneling.sh | 0 .../airbyte-cdk/gcs-destinations/build.gradle | 7 + .../destination/jdbc/copy/gcs/GcsConfig.java | 0 .../jdbc/copy/gcs/GcsStreamCopier.java | 0 .../jdbc/copy/gcs/GcsStreamCopierFactory.java | 0 .../java/airbyte-cdk/init-oss/build.gradle | 5 - .../airbyte-cdk/s3-destinations/build.gradle | 81 +- .../destination/gcs/BaseGcsDestination.java | 19 +- .../destination/gcs/GcsDestinationConfig.java | 8 +- .../destination/gcs/GcsNameTransformer.java | 2 +- .../destination/gcs/GcsStorageOperations.java | 2 +- .../destination/gcs/avro/GcsAvroWriter.java | 13 +- .../gcs/credential/GcsCredentialConfig.java | 2 +- .../gcs/credential/GcsCredentialConfigs.java | 2 +- .../gcs/credential/GcsCredentialType.java | 2 +- .../GcsHmacKeyCredentialConfig.java | 2 +- .../destination/gcs/csv/GcsCsvWriter.java | 6 +- .../destination/gcs/jsonl/GcsJsonlWriter.java | 6 +- .../gcs/parquet/GcsParquetWriter.java | 11 +- .../destination/gcs/util/GcsS3FileSystem.java | 2 +- .../destination/gcs/util/GcsUtils.java | 2 +- .../destination/gcs/writer/BaseGcsWriter.java | 4 +- .../staging/StagingConsumerFactory.java | 5 +- .../gcs/GcsDestinationConfigTest.java | 6 +- .../gcs/avro/GcsAvroFormatConfigTest.java | 6 +- .../gcs/avro/GcsAvroWriterTest.java | 6 +- .../gcs/csv/GcsCsvFormatConfigTest.java | 6 +- .../gcs/jsonl/GcsJsonlFormatConfigTest.java | 6 +- .../destination/gcs/util/ConfigTestUtils.java | 4 +- .../src/test/resources/test_config.json | 0 ...sAvroParquetDestinationAcceptanceTest.java | 2 +- .../gcs/GcsAvroTestDataComparator.java | 2 +- .../GcsBaseAvroDestinationAcceptanceTest.java | 11 +- .../GcsBaseCsvDestinationAcceptanceTest.java | 10 +- ...sBaseCsvGzipDestinationAcceptanceTest.java | 4 +- ...GcsBaseJsonlDestinationAcceptanceTest.java | 6 +- ...aseJsonlGzipDestinationAcceptanceTest.java | 4 +- ...sBaseParquetDestinationAcceptanceTest.java | 17 +- .../gcs/GcsDestinationAcceptanceTest.java | 16 +- ...a => S3AvroParquetTestDataComparator.java} | 2 +- .../S3BaseAvroDestinationAcceptanceTest.java | 2 +- ...3BaseParquetDestinationAcceptanceTest.java | 2 +- .../s3/S3DestinationAcceptanceTest.java | 5 - airbyte-cdk/java/airbyte-cdk/settings.gradle | 14 +- .../airbyte-cdk/typing-deduping/build.gradle | 33 +- .../base/destination/typing_deduping/Sql.java | 5 +- .../bases/base-java/build.gradle | 24 - .../bases/base-normalization/build.gradle | 9 +- .../destination-harness/build.gradle | 3 +- .../source-harness/build.gradle | 2 +- .../destination-bigquery/build.gradle | 41 +- .../bigquery/BigQueryDestination.java | 10 +- .../bigquery/BigQueryGcsOperations.java | 4 +- .../destination/bigquery/BigQueryUtils.java | 2 +- .../BigQueryDestinationHandler.java | 5 + .../bigquery/BigQueryDestinationTest.java | 2 +- .../BigQueryGcsDestinationAcceptanceTest.java | 2 +- .../connectors/destination-gcs/build.gradle | 40 +- .../destination/GcsDestination.java | 18 + .../GcsAvroDestinationAcceptanceTest.java | 16 + .../GcsCsvDestinationAcceptanceTest.java | 15 + .../GcsCsvGzipDestinationAcceptanceTest.java | 15 + .../GcsJsonlDestinationAcceptanceTest.java | 16 + ...GcsJsonlGzipDestinationAcceptanceTest.java | 15 + .../GcsParquetDestinationAcceptanceTest.java | 15 + .../connectors/source-redshift/build.gradle | 3 - build.gradle | 35 +- .../main/groovy/airbyte-java-connector.gradle | 58 +- settings.gradle | 14 +- 378 files changed, 1408 insertions(+), 5023 deletions(-) delete mode 100644 airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/build.gradle delete mode 100644 airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriter.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriterFactory.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-api/readme.md delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/build.gradle delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/readme.md delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/build.gradle delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageVersionedMigrator.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteProtocolVersionedMigratorFactory.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ConfiguredAirbyteCatalogMigrator.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/VersionedProtocolSerializer.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/AirbyteMessageMigration.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/ConfiguredAirbyteCatalogMigration.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/Migration.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/MigrationContainer.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/RecordMigrations.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/SchemaMigrations.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/CatalogMigrationV1Helper.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/SchemaMigrationV1.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageDeserializer.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericDeserializer.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericSerializer.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageSerializer.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Deserializer.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Serializer.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageMigratorTest.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderTest.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1Test.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1Test.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0SerDeTest.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1SerDeTest.java delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/resources/WellKnownTypes.json delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons/LICENSE delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-commons/readme.md delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/LICENSE delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/build.gradle delete mode 100644 airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/readme.md create mode 100644 airbyte-cdk/java/airbyte-cdk/azure-blob-storage-destinations/build.gradle rename airbyte-cdk/java/airbyte-cdk/{db-destinations => azure-blob-storage-destinations}/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageConfig.java (100%) rename airbyte-cdk/java/airbyte-cdk/{db-destinations => azure-blob-storage-destinations}/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopier.java (100%) rename airbyte-cdk/java/airbyte-cdk/{db-destinations => azure-blob-storage-destinations}/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopierFactory.java (100%) delete mode 100644 airbyte-cdk/java/airbyte-cdk/config-models-oss/README.md delete mode 100644 airbyte-cdk/java/airbyte-cdk/config-models-oss/build.gradle rename airbyte-cdk/java/airbyte-cdk/{db-destinations => core}/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperations.java (98%) rename airbyte-cdk/java/airbyte-cdk/{db-destinations => core}/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/StagingFilenameGenerator.java (100%) rename airbyte-cdk/java/airbyte-cdk/{db-destinations => core}/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java (100%) rename airbyte-cdk/java/airbyte-cdk/{db-destinations => core}/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopier.java (100%) rename airbyte-cdk/java/airbyte-cdk/{db-destinations => core}/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopierFactory.java (100%) rename airbyte-cdk/java/airbyte-cdk/{db-destinations => core}/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java (97%) delete mode 100644 airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/pre_migration_schema.txt delete mode 100644 airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema.sql delete mode 100644 airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema_dump.txt rename airbyte-cdk/java/airbyte-cdk/core/src/{main => testFixtures}/java/io/airbyte/cdk/integrations/base/ssh/SshBastionContainer.java (76%) rename airbyte-cdk/java/airbyte-cdk/core/src/{main => testFixtures}/java/io/airbyte/cdk/integrations/util/HostPortResolver.java (84%) rename airbyte-cdk/java/airbyte-cdk/{db-sources => core}/src/testFixtures/java/io/airbyte/cdk/testutils/ContainerFactory.java (100%) rename airbyte-cdk/java/airbyte-cdk/{db-sources => core}/src/testFixtures/java/io/airbyte/cdk/testutils/DatabaseConnectionHelper.java (100%) rename airbyte-cdk/java/airbyte-cdk/{db-sources => core}/src/testFixtures/java/io/airbyte/cdk/testutils/NonContainer.java (100%) rename airbyte-cdk/java/airbyte-cdk/{db-sources => core}/src/testFixtures/java/io/airbyte/cdk/testutils/TestDatabase.java (99%) create mode 100644 airbyte-cdk/java/airbyte-cdk/datastore-bigquery/build.gradle rename airbyte-cdk/java/airbyte-cdk/{core => datastore-bigquery}/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryDatabase.java (100%) rename airbyte-cdk/java/airbyte-cdk/{core => datastore-bigquery}/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryResultSet.java (100%) rename airbyte-cdk/java/airbyte-cdk/{core => datastore-bigquery}/src/main/java/io/airbyte/cdk/db/bigquery/BigQuerySourceOperations.java (100%) rename airbyte-cdk/java/airbyte-cdk/{core => datastore-bigquery}/src/main/java/io/airbyte/cdk/db/bigquery/TempBigQueryJoolDatabaseImpl.java (100%) create mode 100644 airbyte-cdk/java/airbyte-cdk/datastore-mongo/build.gradle create mode 100644 airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoDatabase.java create mode 100644 airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoDatabaseException.java create mode 100644 airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoUtils.java create mode 100644 airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/TreeNode.java create mode 100644 airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/test/java/io/airbyte/cdk/db/mongodb/MongoUtilsTest.java create mode 100644 airbyte-cdk/java/airbyte-cdk/datastore-postgres/build.gradle create mode 100644 airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/java/io/airbyte/cdk/db/PgLsn.java create mode 100644 airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/java/io/airbyte/cdk/db/PostgresUtils.java rename airbyte-cdk/java/airbyte-cdk/{core => datastore-postgres}/src/main/java/io/airbyte/cdk/integrations/util/PostgresSslConnectionUtils.java (100%) rename airbyte-cdk/java/airbyte-cdk/{core => datastore-postgres}/src/test/java/io/airbyte/cdk/db/PgLsnTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{core => datastore-postgres}/src/test/java/io/airbyte/cdk/db/PostgresUtilsTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{db-sources => datastore-postgres}/src/testFixtures/java/io/airbyte/cdk/testutils/PostgreSQLContainerHelper.java (100%) delete mode 100644 airbyte-cdk/java/airbyte-cdk/db-sources/src/main/resources/spec.json delete mode 100644 airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/dummy_config.json delete mode 100644 airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/expected_spec.json rename airbyte-cdk/java/airbyte-cdk/{airbyte-api => dependencies}/build.gradle (64%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-api => dependencies}/src/main/java/io/airbyte/api/client/AirbyteApiClient.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-api => dependencies}/src/main/java/io/airbyte/api/client/PatchedLogsApi.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons-cli => dependencies}/src/main/java/io/airbyte/commons/cli/Clis.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/concurrency/CompletableFutures.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/concurrency/VoidCallable.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/concurrency/WaitingUtils.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/constants/AirbyteSecretConstants.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/enums/Enums.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/exceptions/ConfigErrorException.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/exceptions/ConnectionErrorException.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/exceptions/SQLRuntimeException.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/features/FeatureFlagHelper.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/features/FeatureFlags.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/features/FeatureFlagsWrapper.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/functional/CheckedBiConsumer.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/functional/CheckedBiFunction.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/functional/CheckedConsumer.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/functional/CheckedFunction.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/functional/CheckedSupplier.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/io/IOs.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/io/LineGobbler.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/jackson/MoreMappers.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/json/JsonPaths.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/json/JsonSchemas.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/json/Jsons.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/lang/CloseableConsumer.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/lang/CloseableQueue.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/lang/CloseableShutdownHook.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/lang/Exceptions.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/lang/MoreBooleans.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/logging/LoggingHelper.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/logging/MdcScope.java (97%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/map/MoreMaps.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons-protocol => dependencies}/src/main/java/io/airbyte/commons/protocol/DefaultProtocolSerializer.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons-protocol => dependencies}/src/main/java/io/airbyte/commons/protocol/ProtocolSerializer.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/resources/MoreResources.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/stream/AirbyteStreamStatusHolder.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/stream/AirbyteStreamUtils.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/stream/MoreStreams.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/stream/StreamStatusUtils.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/string/Strings.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/text/Names.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/text/Sqls.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/util/AirbyteStreamAware.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/util/AutoCloseableIterator.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/util/CompositeIterator.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/util/DefaultAutoCloseableIterator.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/util/LazyAutoCloseableIterator.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/util/MoreIterators.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersion.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/version/AirbyteVersion.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/version/Version.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/version/VersionDeserializer.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/version/VersionSerializer.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/java/io/airbyte/commons/yaml/Yamls.java (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/java/io/airbyte/configoss/AirbyteConfig.java (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/java/io/airbyte/configoss/AirbyteConfigValidator.java (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/java/io/airbyte/configoss/CatalogDefinitionsConfig.java (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/java/io/airbyte/configoss/ConfigSchema.java (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/java/io/airbyte/configoss/Configs.java (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/java/io/airbyte/configoss/EnvConfigs.java (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/java/io/airbyte/configoss/helpers/StateMessageHelper.java (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitions.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-json-validation => dependencies}/src/main/java/io/airbyte/validation/json/AbstractSchemaValidator.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-json-validation => dependencies}/src/main/java/io/airbyte/validation/json/ConfigSchemaValidator.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-json-validation => dependencies}/src/main/java/io/airbyte/validation/json/JsonSchemaValidator.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-json-validation => dependencies}/src/main/java/io/airbyte/validation/json/JsonValidationException.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-api => dependencies}/src/main/openapi/config.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/main/resources/log4j2-test.xml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/ActorDefinitionResourceRequirements.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/ActorType.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/AllowedHosts.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/CombinedConnectorCatalog.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/ConnectorJobOutput.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/DataType.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/DestinationConnection.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/DestinationOAuthParameter.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/DestinationSyncMode.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/DockerImageSpec.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/FailureReason.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/JobGetSpecConfig.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/JobSyncConfig.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/JobType.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/NamespaceDefinitionType.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/NotificationType.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/OperatorDbt.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/OperatorNormalization.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/OperatorType.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/OperatorWebhook.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/ReplicationStatus.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/ResourceRequirements.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/SourceConnection.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/SourceOAuthParameter.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/StandardCheckConnectionInput.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/StandardCheckConnectionOutput.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/StandardDestinationDefinition.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/StandardDiscoverCatalogInput.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/StandardSourceDefinition.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/StandardSyncInput.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/StandardSyncOperation.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/State.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/StateType.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/StateWrapper.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/SuggestedStreams.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/SyncMode.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/SyncStats.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/WebhookOperationConfigs.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/WebhookOperationSummary.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/WorkerDestinationConfig.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/main/resources/types/WorkerSourceConfig.yaml (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-api => dependencies}/src/test/java/io/airbyte/api/client/AirbyteApiClientTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons-cli => dependencies}/src/test/java/io/airbyte/commons/cli/ClisTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/concurrency/WaitingUtilsTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/enums/EnumsTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/features/FeatureFlagHelperTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/io/IOsTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/io/LineGobblerTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/json/JsonPathsTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/json/JsonsTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/lang/CloseableShutdownHookTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/lang/ExceptionsTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/lang/MoreBooleansTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/map/MoreMapsTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/stream/AirbyteStreamStatusHolderTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/stream/StreamStatusUtilsTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/string/StringsTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/text/NamesTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/text/SqlsTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/util/CompositeIteratorTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/util/DefaultAutoCloseableIteratorTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/util/LazyAutoCloseableIteratorTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/version/VersionTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/java/io/airbyte/commons/yaml/YamlsTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/test/java/io/airbyte/configoss/ConfigSchemaTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/test/java/io/airbyte/configoss/DataTypeEnumTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{config-models-oss => dependencies}/src/test/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitionsTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-json-validation => dependencies}/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness => dependencies}/src/test/java/io/airbyte/workers/TestHarnessUtilsTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness => dependencies}/src/test/java/io/airbyte/workers/helper/CatalogClientConvertersTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness => dependencies}/src/test/java/io/airbyte/workers/helper/ConnectorConfigUpdaterTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness => dependencies}/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness => dependencies}/src/test/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactoryTest.java (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/resources/json_schemas/composite_json_schema.json (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/resources/json_schemas/json_with_all_types.json (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/resources/json_schemas/json_with_array_type_fields.json (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/resources/json_schemas/json_with_array_type_fields_no_items.json (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/resources/json_schemas/json_with_array_type_fields_with_composites.json (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/resources/resource_test (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/resources/resource_test_a (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/resources/subdir/resource_test_a (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/resources/subdir/resource_test_sub (100%) rename airbyte-cdk/java/airbyte-cdk/{airbyte-commons => dependencies}/src/test/resources/subdir/resource_test_sub_2 (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness => dependencies}/src/test/resources/version-detection/logs-with-version.jsonl (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness => dependencies}/src/test/resources/version-detection/logs-without-spec-message.jsonl (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness => dependencies}/src/test/resources/version-detection/logs-without-version.jsonl (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/TestHarness.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/TestHarnessUtils.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/WorkerConstants.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/exception/TestHarnessException.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/general/CheckConnectionTestHarness.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/general/DbtTransformationRunner.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/general/DefaultCheckConnectionTestHarness.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/general/DefaultDiscoverCatalogTestHarness.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/general/DefaultGetSpecTestHarness.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/general/DiscoverCatalogTestHarness.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/general/GetSpecTestHarness.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/helper/CatalogClientConverters.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/helper/ConnectorConfigUpdater.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/helper/EntrypointEnvChecker.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/helper/FailureHelper.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/internal/AirbyteDestination.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriter.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriterFactory.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/internal/AirbyteProtocolPredicate.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/internal/AirbyteSource.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/internal/AirbyteStreamFactory.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriter.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriterFactory.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/internal/DefaultAirbyteSource.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java (99%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/internal/HeartbeatMonitor.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/normalization/NormalizationAirbyteStreamFactory.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/normalization/NormalizationRunner.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/process/DockerProcessFactory.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/process/IntegrationLauncher.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/process/Metadata.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/process/ProcessFactory.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/java/io/airbyte/workers/test_utils/TestConfigHelpers.java (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/resources/dbt_transformation_entrypoint.sh (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/resources/entrypoints/sync/check.sh (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/resources/entrypoints/sync/init.sh (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/resources/entrypoints/sync/main.sh (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/resources/image_exists.sh (100%) rename airbyte-cdk/java/airbyte-cdk/{acceptance-test-harness/src/main => dependencies/src/testFixtures}/resources/sshtunneling.sh (100%) create mode 100644 airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle rename airbyte-cdk/java/airbyte-cdk/{db-destinations => gcs-destinations}/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsConfig.java (100%) rename airbyte-cdk/java/airbyte-cdk/{db-destinations => gcs-destinations}/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopier.java (100%) rename airbyte-cdk/java/airbyte-cdk/{db-destinations => gcs-destinations}/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopierFactory.java (100%) delete mode 100644 airbyte-cdk/java/airbyte-cdk/init-oss/build.gradle rename airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/BaseGcsDestination.java (83%) rename {airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk}/integrations/destination/gcs/GcsDestinationConfig.java (90%) rename {airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk}/integrations/destination/gcs/GcsNameTransformer.java (79%) rename {airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk}/integrations/destination/gcs/GcsStorageOperations.java (97%) rename {airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk}/integrations/destination/gcs/avro/GcsAvroWriter.java (91%) rename {airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk}/integrations/destination/gcs/credential/GcsCredentialConfig.java (86%) rename {airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk}/integrations/destination/gcs/credential/GcsCredentialConfigs.java (91%) rename {airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk}/integrations/destination/gcs/credential/GcsCredentialType.java (63%) rename {airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk}/integrations/destination/gcs/credential/GcsHmacKeyCredentialConfig.java (95%) rename {airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk}/integrations/destination/gcs/csv/GcsCsvWriter.java (95%) rename {airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk}/integrations/destination/gcs/jsonl/GcsJsonlWriter.java (94%) rename {airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk}/integrations/destination/gcs/parquet/GcsParquetWriter.java (92%) rename {airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk}/integrations/destination/gcs/util/GcsS3FileSystem.java (90%) rename {airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk}/integrations/destination/gcs/util/GcsUtils.java (97%) rename {airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk}/integrations/destination/gcs/writer/BaseGcsWriter.java (97%) rename {airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk}/integrations/destination/gcs/GcsDestinationConfigTest.java (88%) rename {airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk}/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java (96%) rename {airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk}/integrations/destination/gcs/avro/GcsAvroWriterTest.java (88%) rename {airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk}/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java (93%) rename {airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk}/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java (92%) rename {airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk}/integrations/destination/gcs/util/ConfigTestUtils.java (89%) rename {airbyte-integrations/connectors/destination-gcs => airbyte-cdk/java/airbyte-cdk/s3-destinations}/src/test/resources/test_config.json (100%) rename {airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk}/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java (99%) rename {airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk}/integrations/destination/gcs/GcsAvroTestDataComparator.java (97%) rename airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroDestinationAcceptanceTest.java => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseAvroDestinationAcceptanceTest.java (91%) rename airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvDestinationAcceptanceTest.java => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvDestinationAcceptanceTest.java (93%) rename airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvGzipDestinationAcceptanceTest.java => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvGzipDestinationAcceptanceTest.java (86%) rename airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlDestinationAcceptanceTest.java => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlDestinationAcceptanceTest.java (91%) rename airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlGzipDestinationAcceptanceTest.java => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlGzipDestinationAcceptanceTest.java (85%) rename airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsParquetDestinationAcceptanceTest.java => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseParquetDestinationAcceptanceTest.java (88%) rename {airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte => airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk}/integrations/destination/gcs/GcsDestinationAcceptanceTest.java (96%) rename airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/{S3BaseAvroParquetTestDataComparator.java => S3AvroParquetTestDataComparator.java} (94%) create mode 100644 airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/GcsDestination.java create mode 100644 airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsAvroDestinationAcceptanceTest.java create mode 100644 airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsCsvDestinationAcceptanceTest.java create mode 100644 airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsCsvGzipDestinationAcceptanceTest.java create mode 100644 airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsJsonlDestinationAcceptanceTest.java create mode 100644 airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsJsonlGzipDestinationAcceptanceTest.java create mode 100644 airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsParquetDestinationAcceptanceTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md index b8e2addd56a2bb..2e6220c585ce4b 100644 --- a/airbyte-cdk/java/airbyte-cdk/README.md +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -166,6 +166,7 @@ MavenLocal debugging steps: | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.17.0 | 2024-02-01 | [\#34745](https://github.com/airbytehq/airbyte/pull/34745) | Reorganize CDK module structure. | | 0.16.3 | 2024-01-30 | [\#34669](https://github.com/airbytehq/airbyte/pull/34669) | Fix org.apache.logging.log4j:log4j-slf4j-impl version conflicts. | | 0.16.2 | 2024-01-29 | [\#34630](https://github.com/airbytehq/airbyte/pull/34630) | expose NamingTransformer to sub-classes in destinations JdbcSqlGenerator. | | 0.16.1 | 2024-01-29 | [\#34533](https://github.com/airbytehq/airbyte/pull/34533) | Add a safe method to execute DatabaseMetadata's Resultset returning queries. | diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/build.gradle b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/build.gradle deleted file mode 100644 index b2def01ec3eee9..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/build.gradle +++ /dev/null @@ -1,41 +0,0 @@ -plugins { - id "java-library" -} - -java { - compileJava { - options.compilerArgs += "-Xlint:-try" - } -} - -dependencies { - implementation group: 'joda-time', name: 'joda-time', version: '2.12.5' - implementation 'io.fabric8:kubernetes-client:5.12.2' - implementation 'com.auth0:java-jwt:3.19.2' - implementation libs.guava - implementation(libs.temporal.sdk) { - exclude module: 'guava' - } - implementation 'org.apache.ant:ant:1.10.10' - implementation 'org.apache.commons:commons-text:1.10.0' - implementation libs.bundles.datadog - implementation group: 'io.swagger', name: 'swagger-annotations', version: '1.6.2' - - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-api') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-protocol') - implementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - - testAnnotationProcessor libs.jmh.annotations - - testImplementation 'com.jayway.jsonpath:json-path:2.7.0' - testImplementation 'org.mockito:mockito-inline:4.7.0' - testImplementation libs.postgresql - testImplementation libs.testcontainers - testImplementation libs.testcontainers.postgresql - testImplementation libs.jmh.core - testImplementation libs.jmh.annotations - testImplementation 'com.github.docker-java:docker-java:3.2.8' - testImplementation 'com.github.docker-java:docker-java-transport-httpclient5:3.2.8' -} diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriter.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriter.java deleted file mode 100644 index 9d794195273581..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriter.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.workers.internal; - -import io.airbyte.commons.protocol.AirbyteMessageVersionedMigrator; -import io.airbyte.commons.protocol.serde.AirbyteMessageSerializer; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import java.io.BufferedWriter; -import java.io.IOException; -import java.util.Optional; - -public class VersionedAirbyteMessageBufferedWriter extends DefaultAirbyteMessageBufferedWriter { - - private final AirbyteMessageSerializer serializer; - private final AirbyteMessageVersionedMigrator migrator; - private final Optional configuredAirbyteCatalog; - - public VersionedAirbyteMessageBufferedWriter(final BufferedWriter writer, - final AirbyteMessageSerializer serializer, - final AirbyteMessageVersionedMigrator migrator, - final Optional configuredAirbyteCatalog) { - super(writer); - this.serializer = serializer; - this.migrator = migrator; - this.configuredAirbyteCatalog = configuredAirbyteCatalog; - } - - @Override - public void write(final AirbyteMessage message) throws IOException { - final T downgradedMessage = migrator.downgrade(message, configuredAirbyteCatalog); - writer.write(serializer.serialize(downgradedMessage)); - writer.newLine(); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriterFactory.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriterFactory.java deleted file mode 100644 index 32a848f8413121..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriterFactory.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.workers.internal; - -import io.airbyte.commons.protocol.AirbyteMessageSerDeProvider; -import io.airbyte.commons.protocol.AirbyteProtocolVersionedMigratorFactory; -import io.airbyte.commons.version.Version; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import java.io.BufferedWriter; -import java.util.Optional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class VersionedAirbyteMessageBufferedWriterFactory implements AirbyteMessageBufferedWriterFactory { - - private static final Logger LOGGER = LoggerFactory.getLogger(VersionedAirbyteMessageBufferedWriterFactory.class); - - private final AirbyteMessageSerDeProvider serDeProvider; - private final AirbyteProtocolVersionedMigratorFactory migratorFactory; - private final Version protocolVersion; - private final Optional configuredAirbyteCatalog; - - public VersionedAirbyteMessageBufferedWriterFactory(final AirbyteMessageSerDeProvider serDeProvider, - final AirbyteProtocolVersionedMigratorFactory migratorFactory, - final Version protocolVersion, - final Optional configuredAirbyteCatalog) { - this.serDeProvider = serDeProvider; - this.migratorFactory = migratorFactory; - this.protocolVersion = protocolVersion; - this.configuredAirbyteCatalog = configuredAirbyteCatalog; - } - - @Override - public AirbyteMessageBufferedWriter createWriter(BufferedWriter bufferedWriter) { - final boolean needMigration = !protocolVersion.getMajorVersion().equals(migratorFactory.getMostRecentVersion().getMajorVersion()); - LOGGER.info( - "Writing messages to protocol version {}{}", - protocolVersion.serialize(), - needMigration ? ", messages will be downgraded from protocol version " + migratorFactory.getMostRecentVersion().serialize() : ""); - return new VersionedAirbyteMessageBufferedWriter<>( - bufferedWriter, - serDeProvider.getSerializer(protocolVersion).orElseThrow(), - migratorFactory.getAirbyteMessageMigrator(protocolVersion), - configuredAirbyteCatalog); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-api/readme.md b/airbyte-cdk/java/airbyte-cdk/airbyte-api/readme.md deleted file mode 100644 index 33ffeeb918dd70..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-api/readme.md +++ /dev/null @@ -1,11 +0,0 @@ -# airbyte-api - -Defines the OpenApi configuration for the Airbyte Configuration API. It also is responsible for generating the following from the API spec: -* Java API client -* Java API server - this generated code is used in `airbyte-server` to allow us to implement the Configuration API in a type safe way. See `ConfigurationApi.java` in `airbyte-server` -* API docs - -## Key Files -* src/openapi/config.yaml - Defines the config API interface using OpenApi3 -* AirbyteApiClient.java - wraps all api clients so that they can be dependency injected together -* PatchedLogsApi.java - fixes generated code for log api. diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/build.gradle deleted file mode 100644 index 3dbb175d2ad9aa..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/build.gradle +++ /dev/null @@ -1,7 +0,0 @@ -plugins { - id "java-library" -} - -dependencies { - implementation 'commons-cli:commons-cli:1.4' -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/readme.md b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/readme.md deleted file mode 100644 index 81aa7feb0b334b..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/readme.md +++ /dev/null @@ -1,3 +0,0 @@ -# airbyte-commons-cli - -This module houses utility functions for the `commons-cli` library. It is separate from `commons`, because it depends on external library `commons-cli` which we do not want to introduce as a dependency to every module. diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/build.gradle deleted file mode 100644 index 05dc9a3fedb3b1..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/build.gradle +++ /dev/null @@ -1,10 +0,0 @@ -java { - compileJava { - options.compilerArgs += "-Xlint:-unchecked" - } -} - -dependencies { - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java deleted file mode 100644 index 8368cb5128f26d..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import com.google.common.annotations.VisibleForTesting; -import io.airbyte.commons.protocol.migrations.AirbyteMessageMigration; -import io.airbyte.commons.protocol.migrations.MigrationContainer; -import io.airbyte.commons.version.Version; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -/** - * AirbyteProtocol Message Migrator - * - * This class is intended to apply the transformations required to go from one version of the - * AirbyteProtocol to another. - */ -public class AirbyteMessageMigrator { - - private final MigrationContainer> migrationContainer; - - public AirbyteMessageMigrator(final List> migrations) { - migrationContainer = new MigrationContainer<>(migrations); - } - - public void initialize() { - migrationContainer.initialize(); - } - - /** - * Downgrade a message from the most recent version to the target version by chaining all the - * required migrations - */ - public PreviousVersion downgrade(final CurrentVersion message, - final Version target, - final Optional configuredAirbyteCatalog) { - return migrationContainer.downgrade(message, target, (migration, msg) -> applyDowngrade(migration, msg, configuredAirbyteCatalog)); - } - - /** - * Upgrade a message from the source version to the most recent version by chaining all the required - * migrations - */ - public CurrentVersion upgrade(final PreviousVersion message, - final Version source, - final Optional configuredAirbyteCatalog) { - return migrationContainer.upgrade(message, source, (migration, msg) -> applyUpgrade(migration, msg, configuredAirbyteCatalog)); - } - - public Version getMostRecentVersion() { - return migrationContainer.getMostRecentVersion(); - } - - // Helper function to work around type casting - private static PreviousVersion applyDowngrade(final AirbyteMessageMigration migration, - final Object message, - final Optional configuredAirbyteCatalog) { - return migration.downgrade((CurrentVersion) message, configuredAirbyteCatalog); - } - - // Helper function to work around type casting - private static CurrentVersion applyUpgrade(final AirbyteMessageMigration migration, - final Object message, - final Optional configuredAirbyteCatalog) { - return migration.upgrade((PreviousVersion) message, configuredAirbyteCatalog); - } - - // Used for inspection of the injection - @VisibleForTesting - Set getMigrationKeys() { - return migrationContainer.getMigrationKeys(); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java deleted file mode 100644 index f045a8cc4a78e6..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import com.google.common.annotations.VisibleForTesting; -import io.airbyte.commons.protocol.serde.AirbyteMessageDeserializer; -import io.airbyte.commons.protocol.serde.AirbyteMessageSerializer; -import io.airbyte.commons.version.Version; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - -/** - * AirbyteProtocol Message Serializer/Deserializer provider - * - * This class is intended to help access the serializer/deserializer for a given version of the - * Airbyte Protocol. - */ -public class AirbyteMessageSerDeProvider { - - private final List> deserializersToRegister; - private final List> serializersToRegister; - - private final Map> deserializers = new HashMap<>(); - private final Map> serializers = new HashMap<>(); - - public AirbyteMessageSerDeProvider(final List> deserializers, - final List> serializers) { - deserializersToRegister = deserializers; - serializersToRegister = serializers; - } - - public AirbyteMessageSerDeProvider() { - this(Collections.emptyList(), Collections.emptyList()); - } - - public void initialize() { - deserializersToRegister.forEach(this::registerDeserializer); - serializersToRegister.forEach(this::registerSerializer); - } - - /** - * Returns the Deserializer for the version if known else empty - */ - public Optional> getDeserializer(final Version version) { - return Optional.ofNullable(deserializers.get(version.getMajorVersion())); - } - - /** - * Returns the Serializer for the version if known else empty - */ - public Optional> getSerializer(final Version version) { - return Optional.ofNullable(serializers.get(version.getMajorVersion())); - } - - @VisibleForTesting - void registerDeserializer(final AirbyteMessageDeserializer deserializer) { - final String key = deserializer.getTargetVersion().getMajorVersion(); - if (!deserializers.containsKey(key)) { - deserializers.put(key, deserializer); - } else { - throw new RuntimeException(String.format("Trying to register a deserializer for protocol version {} when {} already exists", - deserializer.getTargetVersion().serialize(), deserializers.get(key).getTargetVersion().serialize())); - } - } - - @VisibleForTesting - void registerSerializer(final AirbyteMessageSerializer serializer) { - final String key = serializer.getTargetVersion().getMajorVersion(); - if (!serializers.containsKey(key)) { - serializers.put(key, serializer); - } else { - throw new RuntimeException(String.format("Trying to register a serializer for protocol version {} when {} already exists", - serializer.getTargetVersion().serialize(), serializers.get(key).getTargetVersion().serialize())); - } - } - - // Used for inspection of the injection - @VisibleForTesting - Set getDeserializerKeys() { - return deserializers.keySet(); - } - - // Used for inspection of the injection - @VisibleForTesting - Set getSerializerKeys() { - return serializers.keySet(); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageVersionedMigrator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageVersionedMigrator.java deleted file mode 100644 index f3339fcd5e38c3..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageVersionedMigrator.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import io.airbyte.commons.version.Version; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import java.util.Optional; - -/** - * Wraps message migration from a fixed version to the most recent version - */ -public class AirbyteMessageVersionedMigrator { - - private final AirbyteMessageMigrator migrator; - private final Version version; - - public AirbyteMessageVersionedMigrator(final AirbyteMessageMigrator migrator, final Version version) { - this.migrator = migrator; - this.version = version; - } - - public OriginalMessageType downgrade(final AirbyteMessage message, final Optional configuredAirbyteCatalog) { - return migrator.downgrade(message, version, configuredAirbyteCatalog); - } - - public AirbyteMessage upgrade(final OriginalMessageType message, final Optional configuredAirbyteCatalog) { - return migrator.upgrade(message, version, configuredAirbyteCatalog); - } - - public Version getVersion() { - return version; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteProtocolVersionedMigratorFactory.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteProtocolVersionedMigratorFactory.java deleted file mode 100644 index 905fa8294902d7..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteProtocolVersionedMigratorFactory.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import io.airbyte.commons.version.Version; - -/** - * Factory to build AirbyteMessageVersionedMigrator - */ -public class AirbyteProtocolVersionedMigratorFactory { - - private final AirbyteMessageMigrator airbyteMessageMigrator; - private final ConfiguredAirbyteCatalogMigrator configuredAirbyteCatalogMigrator; - - public AirbyteProtocolVersionedMigratorFactory(final AirbyteMessageMigrator airbyteMessageMigrator, - final ConfiguredAirbyteCatalogMigrator configuredAirbyteCatalogMigrator) { - this.airbyteMessageMigrator = airbyteMessageMigrator; - this.configuredAirbyteCatalogMigrator = configuredAirbyteCatalogMigrator; - } - - public AirbyteMessageVersionedMigrator getAirbyteMessageMigrator(final Version version) { - return new AirbyteMessageVersionedMigrator<>(airbyteMessageMigrator, version); - } - - public final VersionedProtocolSerializer getProtocolSerializer(final Version version) { - return new VersionedProtocolSerializer(configuredAirbyteCatalogMigrator, version); - } - - public Version getMostRecentVersion() { - return airbyteMessageMigrator.getMostRecentVersion(); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ConfiguredAirbyteCatalogMigrator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ConfiguredAirbyteCatalogMigrator.java deleted file mode 100644 index e62a07f4b772e8..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ConfiguredAirbyteCatalogMigrator.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import com.google.common.annotations.VisibleForTesting; -import io.airbyte.commons.protocol.migrations.ConfiguredAirbyteCatalogMigration; -import io.airbyte.commons.protocol.migrations.MigrationContainer; -import io.airbyte.commons.version.Version; -import java.util.List; -import java.util.Set; - -public class ConfiguredAirbyteCatalogMigrator { - - private final MigrationContainer> migrationContainer; - - public ConfiguredAirbyteCatalogMigrator(final List> migrations) { - migrationContainer = new MigrationContainer<>(migrations); - } - - public void initialize() { - migrationContainer.initialize(); - } - - /** - * Downgrade a message from the most recent version to the target version by chaining all the - * required migrations - */ - public PreviousVersion downgrade(final CurrentVersion message, final Version target) { - return migrationContainer.downgrade(message, target, ConfiguredAirbyteCatalogMigrator::applyDowngrade); - } - - /** - * Upgrade a message from the source version to the most recent version by chaining all the required - * migrations - */ - public CurrentVersion upgrade(final PreviousVersion message, final Version source) { - return migrationContainer.upgrade(message, source, ConfiguredAirbyteCatalogMigrator::applyUpgrade); - } - - public Version getMostRecentVersion() { - return migrationContainer.getMostRecentVersion(); - } - - // Helper function to work around type casting - private static PreviousVersion applyDowngrade(final ConfiguredAirbyteCatalogMigration migration, - final Object message) { - return migration.downgrade((CurrentVersion) message); - } - - // Helper function to work around type casting - private static CurrentVersion applyUpgrade(final ConfiguredAirbyteCatalogMigration migration, - final Object message) { - return migration.upgrade((PreviousVersion) message); - } - - // Used for inspection of the injection - @VisibleForTesting - Set getMigrationKeys() { - return migrationContainer.getMigrationKeys(); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/VersionedProtocolSerializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/VersionedProtocolSerializer.java deleted file mode 100644 index db04473eecba9a..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/VersionedProtocolSerializer.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.version.Version; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; - -/** - * Serialize a ConfiguredAirbyteCatalog to the specified version - *

- * This Serializer expects a ConfiguredAirbyteCatalog from the Current version of the platform, - * converts it to the target protocol version before serializing it. - */ -public class VersionedProtocolSerializer implements ProtocolSerializer { - - private final ConfiguredAirbyteCatalogMigrator configuredAirbyteCatalogMigrator; - private final Version protocolVersion; - - public VersionedProtocolSerializer(final ConfiguredAirbyteCatalogMigrator configuredAirbyteCatalogMigrator, final Version protocolVersion) { - this.configuredAirbyteCatalogMigrator = configuredAirbyteCatalogMigrator; - this.protocolVersion = protocolVersion; - } - - @Override - public String serialize(final ConfiguredAirbyteCatalog configuredAirbyteCatalog) { - return Jsons.serialize(configuredAirbyteCatalogMigrator.downgrade(configuredAirbyteCatalog, protocolVersion)); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/AirbyteMessageMigration.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/AirbyteMessageMigration.java deleted file mode 100644 index c49f9db1e665da..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/AirbyteMessageMigration.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations; - -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import java.util.Optional; - -/** - * AirbyteProtocol message migration interface - * - * @param The Old AirbyteMessage type - * @param The New AirbyteMessage type - */ -public interface AirbyteMessageMigration extends Migration { - - /** - * Downgrades a message to from the new version to the old version - * - * @param message: the message to downgrade - * @param configuredAirbyteCatalog: the ConfiguredAirbyteCatalog of the connection when applicable - * @return the downgraded message - */ - PreviousVersion downgrade(final CurrentVersion message, final Optional configuredAirbyteCatalog); - - /** - * Upgrades a message from the old version to the new version - * - * @param message: the message to upgrade - * @param configuredAirbyteCatalog: the ConfiguredAirbyteCatalog of the connection when applicable - * @return the upgrade message - */ - CurrentVersion upgrade(final PreviousVersion message, final Optional configuredAirbyteCatalog); - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/ConfiguredAirbyteCatalogMigration.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/ConfiguredAirbyteCatalogMigration.java deleted file mode 100644 index 4634c4464e7c04..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/ConfiguredAirbyteCatalogMigration.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations; - -public interface ConfiguredAirbyteCatalogMigration extends Migration { - - /** - * Downgrades a ConfiguredAirbyteCatalog from the new version to the old version - * - * @param message: the ConfiguredAirbyteCatalog to downgrade - * @return the downgraded ConfiguredAirbyteCatalog - */ - PreviousVersion downgrade(final CurrentVersion message); - - /** - * Upgrades a ConfiguredAirbyteCatalog from the old version to the new version - * - * @param message: the ConfiguredAirbyteCatalog to upgrade - * @return the upgraded ConfiguredAirbyteCatalog - */ - CurrentVersion upgrade(final PreviousVersion message); - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/Migration.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/Migration.java deleted file mode 100644 index 47b1469b1a0e13..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/Migration.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations; - -import io.airbyte.commons.version.Version; - -public interface Migration { - - /** - * The Old version, note that due to semver, the important piece of information is the Major. - */ - Version getPreviousVersion(); - - /** - * The New version, note that due to semver, the important piece of information is the Major. - */ - Version getCurrentVersion(); - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/MigrationContainer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/MigrationContainer.java deleted file mode 100644 index c421742d5f3857..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/MigrationContainer.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations; - -import io.airbyte.commons.version.Version; -import java.util.Collection; -import java.util.List; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.function.BiFunction; - -public class MigrationContainer { - - private final List migrationsToRegister; - private final SortedMap migrations = new TreeMap<>(); - - // mostRecentMajorVersion defaults to v0 as no migration is required - private String mostRecentMajorVersion = "0"; - - public MigrationContainer(final List migrations) { - this.migrationsToRegister = migrations; - } - - public void initialize() { - migrationsToRegister.forEach(this::registerMigration); - } - - public Version getMostRecentVersion() { - return new Version(mostRecentMajorVersion, "0", "0"); - } - - /** - * Downgrade a message from the most recent version to the target version by chaining all the - * required migrations - */ - public PreviousVersion downgrade(final CurrentVersion message, - final Version target, - final BiFunction applyDowngrade) { - if (target.getMajorVersion().equals(mostRecentMajorVersion)) { - return (PreviousVersion) message; - } - - Object result = message; - Object[] selectedMigrations = selectMigrations(target).toArray(); - for (int i = selectedMigrations.length; i > 0; --i) { - result = applyDowngrade.apply((T) selectedMigrations[i - 1], result); - } - return (PreviousVersion) result; - } - - /** - * Upgrade a message from the source version to the most recent version by chaining all the required - * migrations - */ - public CurrentVersion upgrade(final PreviousVersion message, - final Version source, - final BiFunction applyUpgrade) { - if (source.getMajorVersion().equals(mostRecentMajorVersion)) { - return (CurrentVersion) message; - } - - Object result = message; - for (var migration : selectMigrations(source)) { - result = applyUpgrade.apply(migration, result); - } - return (CurrentVersion) result; - } - - public Collection selectMigrations(final Version version) { - final Collection results = migrations.tailMap(version.getMajorVersion()).values(); - if (results.isEmpty()) { - throw new RuntimeException("Unsupported migration version " + version.serialize()); - } - return results; - } - - /** - * Store migration in a sorted map key by the major of the lower version of the migration. - * - * The goal is to be able to retrieve the list of migrations to apply to get to/from a given - * version. We are only keying on the lower version because the right side (most recent version of - * the migration range) is always current version. - */ - private void registerMigration(final T migration) { - final String key = migration.getPreviousVersion().getMajorVersion(); - if (!migrations.containsKey(key)) { - migrations.put(key, migration); - if (migration.getCurrentVersion().getMajorVersion().compareTo(mostRecentMajorVersion) > 0) { - mostRecentMajorVersion = migration.getCurrentVersion().getMajorVersion(); - } - } else { - throw new RuntimeException("Trying to register a duplicated migration " + migration.getClass().getName()); - } - } - - public Set getMigrationKeys() { - return migrations.keySet(); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/RecordMigrations.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/RecordMigrations.java deleted file mode 100644 index 4c580929f5af29..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/RecordMigrations.java +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations.util; - -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.ARRAY_TYPE; -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.ITEMS_KEY; -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.OBJECT_TYPE; -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.ONEOF_KEY; -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.PROPERTIES_KEY; -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.REF_KEY; -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.TYPE_KEY; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.validation.json.JsonSchemaValidator; -import java.util.Iterator; -import java.util.Map.Entry; -import java.util.function.BiFunction; -import java.util.function.Function; - -public class RecordMigrations { - - /** - * Quick and dirty tuple. Used internally by - * {@link #mutateDataNode(JsonSchemaValidator, Function, Transformer, JsonNode, JsonNode)}; callers - * probably only actually need the node. - * - * matchedSchema is useful for mutating using a oneOf schema, where we need to recognize the correct - * subschema. - * - * @param node Our attempt at mutating the node, under the given schema - * @param matchedSchema Whether the original node actually matched the schema - */ - public record MigratedNode(JsonNode node, boolean matchedSchema) {} - - /** - * Extend BiFunction so that we can have named parameters. - */ - @FunctionalInterface - public interface Transformer extends BiFunction { - - @Override - MigratedNode apply(JsonNode schema, JsonNode data); - - } - - /** - * Works on a best-effort basis. If the schema doesn't match the data, we'll do our best to mutate - * anything that we can definitively say matches the criteria. Should _not_ throw an exception if - * bad things happen (e.g. we try to parse a non-numerical string as a number). - * - * @param schemaMatcher Accepts a JsonNode schema and returns whether its corresponding entry in the - * data should be mutated. Doesn't need to handle oneOf cases, i.e. should only care about - * type/$ref. - * @param transformer Performs the modification on the given data node. Should not throw exceptions. - */ - public static MigratedNode mutateDataNode( - final JsonSchemaValidator validator, - final Function schemaMatcher, - final Transformer transformer, - final JsonNode data, - final JsonNode schema) { - // If this is a oneOf node, then we need to handle each oneOf case. - if (!schema.hasNonNull(REF_KEY) && !schema.hasNonNull(TYPE_KEY) && schema.hasNonNull(ONEOF_KEY)) { - return mutateOneOfNode(validator, schemaMatcher, transformer, data, schema); - } - - // If we should mutate the data, then mutate it appropriately - if (schemaMatcher.apply(schema)) { - return transformer.apply(schema, data); - } - - // Otherwise, we need to recurse into non-primitive nodes. - if (data.isObject()) { - return mutateObjectNode(validator, schemaMatcher, transformer, data, schema); - } else if (data.isArray()) { - return mutateArrayNode(validator, schemaMatcher, transformer, data, schema); - } else { - // There's nothing to do in the case of a primitive node. - // So we just check whether the schema is correct and return the node as-is. - return new MigratedNode(data, validator.test(schema, data)); - } - } - - /** - * Attempt to mutate using each oneOf option in sequence. Returns the result from mutating using the - * first subschema that matches the data, or if none match, then the result of using the first - * subschema. - */ - private static MigratedNode mutateOneOfNode( - final JsonSchemaValidator validator, - final Function schemaMatcher, - final Transformer transformer, - final JsonNode data, - final JsonNode schema) { - final JsonNode schemaOptions = schema.get(ONEOF_KEY); - if (schemaOptions.size() == 0) { - // If the oneOf has no options, then don't do anything interesting. - return new MigratedNode(data, validator.test(schema, data)); - } - - // Attempt to mutate the node against each oneOf schema. - // Return the first schema that matches the data, or the first schema if none matched successfully. - MigratedNode migratedNode = null; - for (final JsonNode maybeSchema : schemaOptions) { - final MigratedNode maybeMigratedNode = mutateDataNode(validator, schemaMatcher, transformer, data, maybeSchema); - if (maybeMigratedNode.matchedSchema()) { - // If we've found a matching schema, then return immediately - return maybeMigratedNode; - } else if (migratedNode == null) { - // Otherwise - if this is the first subschema, then just take it - migratedNode = maybeMigratedNode; - } - } - // None of the schemas matched, so just return whatever we found first - return migratedNode; - } - - /** - * If data is an object, then we need to recursively mutate all of its fields. - */ - private static MigratedNode mutateObjectNode( - final JsonSchemaValidator validator, - final Function schemaMatcher, - final Transformer transformer, - final JsonNode data, - final JsonNode schema) { - boolean isObjectSchema; - // First, check whether the schema is supposed to be an object at all. - if (schema.hasNonNull(REF_KEY)) { - // If the schema uses a reference type, then it's not an object schema. - isObjectSchema = false; - } else if (schema.hasNonNull(TYPE_KEY)) { - // If the schema declares {type: object} or {type: [..., object, ...]} - // Then this is an object schema - final JsonNode typeNode = schema.get(TYPE_KEY); - if (typeNode.isArray()) { - isObjectSchema = false; - for (final JsonNode typeItem : typeNode) { - if (OBJECT_TYPE.equals(typeItem.asText())) { - isObjectSchema = true; - } - } - } else { - isObjectSchema = OBJECT_TYPE.equals(typeNode.asText()); - } - } else { - // If the schema doesn't declare a type at all (which is bad practice, but let's handle it anyway) - // Then check for a properties entry, and assume that this is an object if it's present - isObjectSchema = schema.hasNonNull(PROPERTIES_KEY); - } - - if (!isObjectSchema) { - // If it's not supposed to be an object, then we can't do anything here. - // Return the data without modification. - return new MigratedNode(data, false); - } else { - // If the schema _is_ for an object, then recurse into each field - final ObjectNode mutatedData = (ObjectNode) Jsons.emptyObject(); - final JsonNode propertiesNode = schema.get(PROPERTIES_KEY); - - final Iterator> dataFields = data.fields(); - boolean matchedSchema = true; - while (dataFields.hasNext()) { - final Entry field = dataFields.next(); - final String key = field.getKey(); - final JsonNode value = field.getValue(); - if (propertiesNode != null && propertiesNode.hasNonNull(key)) { - // If we have a schema for this property, mutate the value - final JsonNode subschema = propertiesNode.get(key); - final MigratedNode migratedNode = mutateDataNode(validator, schemaMatcher, transformer, value, subschema); - mutatedData.set(key, migratedNode.node); - if (!migratedNode.matchedSchema) { - matchedSchema = false; - } - } else { - // Else it's an additional property - we _could_ check additionalProperties, - // but that's annoying. We don't actually respect that in destinations/normalization anyway. - mutatedData.set(key, value); - } - } - - return new MigratedNode(mutatedData, matchedSchema); - } - } - - /** - * Much like objects, arrays must be recursively mutated. - */ - private static MigratedNode mutateArrayNode( - final JsonSchemaValidator validator, - final Function schemaMatcher, - final Transformer transformer, - final JsonNode data, - final JsonNode schema) { - // Similar to objects, we first check whether this is even supposed to be an array. - boolean isArraySchema; - if (schema.hasNonNull(REF_KEY)) { - // If the schema uses a reference type, then it's not an array schema. - isArraySchema = false; - } else if (schema.hasNonNull(TYPE_KEY)) { - // If the schema declares {type: array} or {type: [..., array, ...]} - // Then this is an array schema - final JsonNode typeNode = schema.get(TYPE_KEY); - if (typeNode.isArray()) { - isArraySchema = false; - for (final JsonNode typeItem : typeNode) { - if (ARRAY_TYPE.equals(typeItem.asText())) { - isArraySchema = true; - } - } - } else { - isArraySchema = ARRAY_TYPE.equals(typeNode.asText()); - } - } else { - // If the schema doesn't declare a type at all (which is bad practice, but let's handle it anyway) - // Then check for an items entry, and assume that this is an array if it's present - isArraySchema = schema.hasNonNull(ITEMS_KEY); - } - - if (!isArraySchema) { - return new MigratedNode(data, false); - } else { - final ArrayNode mutatedItems = Jsons.arrayNode(); - final JsonNode itemsNode = schema.get(ITEMS_KEY); - if (itemsNode == null) { - // We _could_ check additionalItems, but much like the additionalProperties comment for objects: - // it's a lot of work for no payoff - return new MigratedNode(data, true); - } else if (itemsNode.isArray()) { - // In the case of {items: [schema1, schema2, ...]} - // We need to check schema1 against the first element of the array, - // schema2 against the second element, etc. - boolean allSchemasMatched = true; - for (int i = 0; i < data.size(); i++) { - final JsonNode element = data.get(i); - if (itemsNode.size() > i) { - // If we have a schema for this element, then try mutating the element - final MigratedNode mutatedElement = mutateDataNode(validator, schemaMatcher, transformer, element, itemsNode.get(i)); - if (!mutatedElement.matchedSchema()) { - allSchemasMatched = false; - } - mutatedItems.add(mutatedElement.node()); - } - } - // If there were more elements in `data` than there were schemas in `itemsNode`, - // then just blindly add the rest of those elements. - for (int i = itemsNode.size(); i < data.size(); i++) { - mutatedItems.add(data.get(i)); - } - return new MigratedNode(mutatedItems, allSchemasMatched); - } else { - // IN the case of {items: schema}, we just check every array element against that schema. - boolean matchedSchema = true; - for (final JsonNode item : data) { - final MigratedNode migratedNode = mutateDataNode(validator, schemaMatcher, transformer, item, itemsNode); - mutatedItems.add(migratedNode.node); - if (!migratedNode.matchedSchema) { - matchedSchema = false; - } - } - return new MigratedNode(mutatedItems, matchedSchema); - } - } - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/SchemaMigrations.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/SchemaMigrations.java deleted file mode 100644 index f689600fdabf94..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/SchemaMigrations.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations.util; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map.Entry; -import java.util.function.Consumer; -import java.util.function.Function; - -/** - * Utility class for recursively modifying JsonSchemas. Useful for up/downgrading AirbyteCatalog - * objects. - * - * See {@link io.airbyte.commons.protocol.migrations.v1.SchemaMigrationV1} for example usage. - */ -public class SchemaMigrations { - - /** - * Generic utility method that recurses through all type declarations in the schema. For each type - * declaration that are accepted by matcher, mutate them using transformer. For all other type - * declarations, recurse into their subschemas (if any). - *

- * Note that this modifies the schema in-place. Callers who need a copy of the old schema should - * save schema.deepCopy() before calling this method. - * - * @param schema The JsonSchema node to walk down - * @param matcher A function which returns true on any schema node that needs to be transformed - * @param transformer A function which mutates a schema node - */ - public static void mutateSchemas(final Function matcher, final Consumer transformer, final JsonNode schema) { - if (schema.isBoolean()) { - // We never want to modify a schema of `true` or `false` (e.g. additionalProperties: true) - // so just return immediately - return; - } - if (matcher.apply(schema)) { - // Base case: If this schema should be mutated, then we need to mutate it - transformer.accept(schema); - } else { - // Otherwise, we need to find all the subschemas and mutate them. - // technically, it might be more correct to do something like: - // if schema["type"] == "array": find subschemas for items, additionalItems, contains - // else if schema["type"] == "object": find subschemas for properties, patternProperties, - // additionalProperties - // else if oneof, allof, etc - // but that sounds really verbose for no real benefit - final List subschemas = findSubschemas(schema); - - // recurse into each subschema - for (final JsonNode subschema : subschemas) { - mutateSchemas(matcher, transformer, subschema); - } - } - } - - /** - * Returns a list of all the direct children nodes to consider for subSchemas - * - * @param schema The JsonSchema node to start - * @return a list of the JsonNodes to be considered - */ - public static List findSubschemas(final JsonNode schema) { - final List subschemas = new ArrayList<>(); - - // array schemas - findSubschemas(subschemas, schema, "items"); - findSubschemas(subschemas, schema, "additionalItems"); - findSubschemas(subschemas, schema, "contains"); - - // object schemas - if (schema.hasNonNull("properties")) { - final ObjectNode propertiesNode = (ObjectNode) schema.get("properties"); - final Iterator> propertiesIterator = propertiesNode.fields(); - while (propertiesIterator.hasNext()) { - final Entry property = propertiesIterator.next(); - subschemas.add(property.getValue()); - } - } - if (schema.hasNonNull("patternProperties")) { - final ObjectNode propertiesNode = (ObjectNode) schema.get("patternProperties"); - final Iterator> propertiesIterator = propertiesNode.fields(); - while (propertiesIterator.hasNext()) { - final Entry property = propertiesIterator.next(); - subschemas.add(property.getValue()); - } - } - findSubschemas(subschemas, schema, "additionalProperties"); - - // combining restrictions - destinations have limited support for these, but we should handle the - // schemas correctly anyway - findSubschemas(subschemas, schema, "allOf"); - findSubschemas(subschemas, schema, "oneOf"); - findSubschemas(subschemas, schema, "anyOf"); - findSubschemas(subschemas, schema, "not"); - - return subschemas; - } - - /** - * If schema contains key, then grab the subschema(s) at schema[key] and add them to the subschemas - * list. - *

- * For example: - *

    - *
  • schema = {"items": [{"type": "string}]} - *

    - * key = "items" - *

    - * -> add {"type": "string"} to subschemas

    • - *
  • schema = {"items": {"type": "string"}} - *

    - * key = "items" - *

    - * -> add {"type": "string"} to subschemas

    • - *
  • schema = {"additionalProperties": true} - *

    - * key = "additionalProperties" - *

    - * -> add nothing to subschemas - *

    - * (technically `true` is a valid JsonSchema, but we don't want to modify it)

    • - *
- */ - public static void findSubschemas(final List subschemas, final JsonNode schema, final String key) { - if (schema.hasNonNull(key)) { - final JsonNode subschemaNode = schema.get(key); - if (subschemaNode.isArray()) { - for (final JsonNode subschema : subschemaNode) { - subschemas.add(subschema); - } - } else if (subschemaNode.isObject()) { - subschemas.add(subschemaNode); - } - } - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1.java deleted file mode 100644 index 76c14cdc54a0c7..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations.v1; - -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.REF_KEY; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; -import com.google.common.annotations.VisibleForTesting; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.protocol.migrations.AirbyteMessageMigration; -import io.airbyte.commons.protocol.migrations.util.RecordMigrations; -import io.airbyte.commons.protocol.migrations.util.RecordMigrations.MigratedNode; -import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.commons.version.Version; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.AirbyteMessage.Type; -import io.airbyte.protocol.models.AirbyteStream; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.JsonSchemaReferenceTypes; -import io.airbyte.validation.json.JsonSchemaValidator; -import java.util.Iterator; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.Optional; - -// Disable V1 Migration, uncomment to re-enable -// @Singleton -public class AirbyteMessageMigrationV1 implements AirbyteMessageMigration { - - private final JsonSchemaValidator validator; - - public AirbyteMessageMigrationV1() { - this(new JsonSchemaValidator()); - } - - @VisibleForTesting - public AirbyteMessageMigrationV1(final JsonSchemaValidator validator) { - this.validator = validator; - } - - @Override - public io.airbyte.protocol.models.v0.AirbyteMessage downgrade(final AirbyteMessage oldMessage, - final Optional configuredAirbyteCatalog) { - final io.airbyte.protocol.models.v0.AirbyteMessage newMessage = Jsons.object( - Jsons.jsonNode(oldMessage), - io.airbyte.protocol.models.v0.AirbyteMessage.class); - if (oldMessage.getType() == Type.CATALOG && oldMessage.getCatalog() != null) { - for (final io.airbyte.protocol.models.v0.AirbyteStream stream : newMessage.getCatalog().getStreams()) { - final JsonNode schema = stream.getJsonSchema(); - SchemaMigrationV1.downgradeSchema(schema); - } - } else if (oldMessage.getType() == Type.RECORD && oldMessage.getRecord() != null) { - if (configuredAirbyteCatalog.isPresent()) { - final ConfiguredAirbyteCatalog catalog = configuredAirbyteCatalog.get(); - final io.airbyte.protocol.models.v0.AirbyteRecordMessage record = newMessage.getRecord(); - final Optional maybeStream = catalog.getStreams().stream() - .filter(stream -> Objects.equals(stream.getStream().getName(), record.getStream()) - && Objects.equals(stream.getStream().getNamespace(), record.getNamespace())) - .findFirst(); - // If this record doesn't belong to any configured stream, then there's no point downgrading it - // So only do the downgrade if we can find its stream - if (maybeStream.isPresent()) { - final JsonNode schema = maybeStream.get().getStream().getJsonSchema(); - final JsonNode oldData = record.getData(); - final MigratedNode downgradedNode = downgradeRecord(oldData, schema); - record.setData(downgradedNode.node()); - } - } - } - return newMessage; - } - - @Override - public AirbyteMessage upgrade(final io.airbyte.protocol.models.v0.AirbyteMessage oldMessage, - final Optional configuredAirbyteCatalog) { - // We're not introducing any changes to the structure of the record/catalog - // so just clone a new message object, which we can edit in-place - final AirbyteMessage newMessage = Jsons.object( - Jsons.jsonNode(oldMessage), - AirbyteMessage.class); - if (oldMessage.getType() == io.airbyte.protocol.models.v0.AirbyteMessage.Type.CATALOG && oldMessage.getCatalog() != null) { - for (final AirbyteStream stream : newMessage.getCatalog().getStreams()) { - final JsonNode schema = stream.getJsonSchema(); - SchemaMigrationV1.upgradeSchema(schema); - } - } else if (oldMessage.getType() == io.airbyte.protocol.models.v0.AirbyteMessage.Type.RECORD && oldMessage.getRecord() != null) { - final JsonNode oldData = newMessage.getRecord().getData(); - final JsonNode newData = upgradeRecord(oldData); - newMessage.getRecord().setData(newData); - } - return newMessage; - } - - /** - * Returns a copy of oldData, with numeric values converted to strings. String and boolean values - * are returned as-is for convenience, i.e. this is not a true deep copy. - */ - private static JsonNode upgradeRecord(final JsonNode oldData) { - if (oldData.isNumber()) { - // Base case: convert numbers to strings - return Jsons.convertValue(oldData.asText(), TextNode.class); - } else if (oldData.isObject()) { - // Recurse into each field of the object - final ObjectNode newData = (ObjectNode) Jsons.emptyObject(); - - final Iterator> fieldsIterator = oldData.fields(); - while (fieldsIterator.hasNext()) { - final Entry next = fieldsIterator.next(); - final String key = next.getKey(); - final JsonNode value = next.getValue(); - - final JsonNode newValue = upgradeRecord(value); - newData.set(key, newValue); - } - - return newData; - } else if (oldData.isArray()) { - // Recurse into each element of the array - final ArrayNode newData = Jsons.arrayNode(); - for (final JsonNode element : oldData) { - newData.add(upgradeRecord(element)); - } - return newData; - } else { - // Base case: this is a string or boolean, so we don't need to modify it - return oldData; - } - } - - /** - * We need the schema to recognize which fields are integers, since it would be wrong to just assume - * any numerical string should be parsed out. - * - * Works on a best-effort basis. If the schema doesn't match the data, we'll do our best to - * downgrade anything that we can definitively say is a number. Should _not_ throw an exception if - * bad things happen (e.g. we try to parse a non-numerical string as a number). - */ - private MigratedNode downgradeRecord(final JsonNode data, final JsonNode schema) { - return RecordMigrations.mutateDataNode( - validator, - s -> { - if (s.hasNonNull(REF_KEY)) { - final String type = s.get(REF_KEY).asText(); - return JsonSchemaReferenceTypes.INTEGER_REFERENCE.equals(type) - || JsonSchemaReferenceTypes.NUMBER_REFERENCE.equals(type); - } else { - return false; - } - }, - (s, d) -> { - if (d.asText().matches("-?\\d+(\\.\\d+)?")) { - // If this string is a numeric literal, convert it to a numeric node. - return new MigratedNode(Jsons.deserialize(d.asText()), true); - } else { - // Otherwise, just leave the node unchanged. - return new MigratedNode(d, false); - } - }, - data, schema); - } - - @Override - public Version getPreviousVersion() { - return AirbyteProtocolVersion.V0; - } - - @Override - public Version getCurrentVersion() { - return AirbyteProtocolVersion.V1; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/CatalogMigrationV1Helper.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/CatalogMigrationV1Helper.java deleted file mode 100644 index fb627ef422f422..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/CatalogMigrationV1Helper.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations.v1; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.protocol.migrations.util.SchemaMigrations; -import io.airbyte.protocol.models.AirbyteCatalog; -import io.airbyte.protocol.models.AirbyteStream; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; - -/** - * For the v0 to v1 migration, it appears that we are persisting some protocol objects without - * version. Until this gets addressed more properly, this class contains the helper functions used - * to handle this on the fly migration. - * - * Once persisted objects are versioned, this code should be deleted. - */ -public class CatalogMigrationV1Helper { - - /** - * Performs an in-place migration of the schema from v0 to v1 if v0 data types are detected - * - * @param configuredAirbyteCatalog to migrate - */ - public static void upgradeSchemaIfNeeded(final ConfiguredAirbyteCatalog configuredAirbyteCatalog) { - if (containsV0DataTypes(configuredAirbyteCatalog)) { - upgradeSchema(configuredAirbyteCatalog); - } - } - - /** - * Performs an in-place migration of the schema from v0 to v1 if v0 data types are detected - * - * @param airbyteCatalog to migrate - */ - public static void upgradeSchemaIfNeeded(final AirbyteCatalog airbyteCatalog) { - if (containsV0DataTypes(airbyteCatalog)) { - upgradeSchema(airbyteCatalog); - } - } - - /** - * Performs an in-place migration of the schema from v0 to v1 - * - * @param configuredAirbyteCatalog to migrate - */ - private static void upgradeSchema(final ConfiguredAirbyteCatalog configuredAirbyteCatalog) { - for (final var stream : configuredAirbyteCatalog.getStreams()) { - SchemaMigrationV1.upgradeSchema(stream.getStream().getJsonSchema()); - } - } - - /** - * Performs an in-place migration of the schema from v0 to v1 - * - * @param airbyteCatalog to migrate - */ - private static void upgradeSchema(final AirbyteCatalog airbyteCatalog) { - for (final var stream : airbyteCatalog.getStreams()) { - SchemaMigrationV1.upgradeSchema(stream.getJsonSchema()); - } - } - - /** - * Returns true if catalog contains v0 data types - */ - private static boolean containsV0DataTypes(final ConfiguredAirbyteCatalog configuredAirbyteCatalog) { - if (configuredAirbyteCatalog == null) { - return false; - } - - return configuredAirbyteCatalog - .getStreams() - .stream().findFirst() - .map(ConfiguredAirbyteStream::getStream) - .map(CatalogMigrationV1Helper::streamContainsV0DataTypes) - .orElse(false); - } - - /** - * Returns true if catalog contains v0 data types - */ - private static boolean containsV0DataTypes(final AirbyteCatalog airbyteCatalog) { - if (airbyteCatalog == null) { - return false; - } - - return airbyteCatalog - .getStreams() - .stream().findFirst() - .map(CatalogMigrationV1Helper::streamContainsV0DataTypes) - .orElse(false); - } - - private static boolean streamContainsV0DataTypes(final AirbyteStream airbyteStream) { - if (airbyteStream == null || airbyteStream.getJsonSchema() == null) { - return false; - } - return hasV0DataType(airbyteStream.getJsonSchema()); - } - - /** - * Performs of search of a v0 data type node, returns true at the first node found. - */ - private static boolean hasV0DataType(final JsonNode schema) { - if (SchemaMigrationV1.isPrimitiveTypeDeclaration(schema)) { - return true; - } - - for (final JsonNode subSchema : SchemaMigrations.findSubschemas(schema)) { - if (hasV0DataType(subSchema)) { - return true; - } - } - return false; - } - - /** - * Performs an in-place migration of the schema from v1 to v0 if v1 data types are detected - * - * @param configuredAirbyteCatalog to migrate - */ - public static void downgradeSchemaIfNeeded(final ConfiguredAirbyteCatalog configuredAirbyteCatalog) { - if (containsV1DataTypes(configuredAirbyteCatalog)) { - downgradeSchema(configuredAirbyteCatalog); - } - } - - /** - * Performs an in-place migration of the schema from v1 to v0 if v1 data types are detected - * - * @param airbyteCatalog to migrate - */ - public static void downgradeSchemaIfNeeded(final AirbyteCatalog airbyteCatalog) { - if (containsV1DataTypes(airbyteCatalog)) { - downgradeSchema(airbyteCatalog); - } - } - - /** - * Performs an in-place migration of the schema from v1 to v0 - * - * @param configuredAirbyteCatalog to migrate - */ - private static void downgradeSchema(final ConfiguredAirbyteCatalog configuredAirbyteCatalog) { - for (final var stream : configuredAirbyteCatalog.getStreams()) { - SchemaMigrationV1.downgradeSchema(stream.getStream().getJsonSchema()); - } - } - - /** - * Performs an in-place migration of the schema from v1 to v0 - * - * @param airbyteCatalog to migrate - */ - private static void downgradeSchema(final AirbyteCatalog airbyteCatalog) { - for (final var stream : airbyteCatalog.getStreams()) { - SchemaMigrationV1.downgradeSchema(stream.getJsonSchema()); - } - } - - /** - * Returns true if catalog contains v1 data types - */ - private static boolean containsV1DataTypes(final ConfiguredAirbyteCatalog configuredAirbyteCatalog) { - if (configuredAirbyteCatalog == null) { - return false; - } - - return configuredAirbyteCatalog - .getStreams() - .stream().findFirst() - .map(ConfiguredAirbyteStream::getStream) - .map(CatalogMigrationV1Helper::streamContainsV1DataTypes) - .orElse(false); - } - - /** - * Returns true if catalog contains v1 data types - */ - private static boolean containsV1DataTypes(final AirbyteCatalog airbyteCatalog) { - if (airbyteCatalog == null) { - return false; - } - - return airbyteCatalog - .getStreams() - .stream().findFirst() - .map(CatalogMigrationV1Helper::streamContainsV1DataTypes) - .orElse(false); - } - - private static boolean streamContainsV1DataTypes(final AirbyteStream airbyteStream) { - if (airbyteStream == null || airbyteStream.getJsonSchema() == null) { - return false; - } - return hasV1DataType(airbyteStream.getJsonSchema()); - } - - /** - * Performs of search of a v0 data type node, returns true at the first node found. - */ - private static boolean hasV1DataType(final JsonNode schema) { - if (SchemaMigrationV1.isPrimitiveReferenceTypeDeclaration(schema)) { - return true; - } - - for (final JsonNode subSchema : SchemaMigrations.findSubschemas(schema)) { - if (hasV1DataType(subSchema)) { - return true; - } - } - return false; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1.java deleted file mode 100644 index 2f56e823b247e8..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations.v1; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.protocol.migrations.ConfiguredAirbyteCatalogMigration; -import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.commons.version.Version; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; - -// Disable V1 Migration, uncomment to re-enable -// @Singleton -public class ConfiguredAirbyteCatalogMigrationV1 - implements ConfiguredAirbyteCatalogMigration { - - @Override - public io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog downgrade(final ConfiguredAirbyteCatalog oldMessage) { - final io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog newMessage = Jsons.object( - Jsons.jsonNode(oldMessage), - io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog.class); - for (final io.airbyte.protocol.models.v0.ConfiguredAirbyteStream stream : newMessage.getStreams()) { - final JsonNode schema = stream.getStream().getJsonSchema(); - SchemaMigrationV1.downgradeSchema(schema); - } - return newMessage; - } - - @Override - public ConfiguredAirbyteCatalog upgrade(final io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog oldMessage) { - final ConfiguredAirbyteCatalog newMessage = Jsons.object( - Jsons.jsonNode(oldMessage), - ConfiguredAirbyteCatalog.class); - for (final ConfiguredAirbyteStream stream : newMessage.getStreams()) { - final JsonNode schema = stream.getStream().getJsonSchema(); - SchemaMigrationV1.upgradeSchema(schema); - } - return newMessage; - } - - @Override - public Version getPreviousVersion() { - return AirbyteProtocolVersion.V0; - } - - @Override - public Version getCurrentVersion() { - return AirbyteProtocolVersion.V1; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/SchemaMigrationV1.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/SchemaMigrationV1.java deleted file mode 100644 index e60fec3a571ca1..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/SchemaMigrationV1.java +++ /dev/null @@ -1,306 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations.v1; - -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.ONEOF_KEY; -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.REF_KEY; -import static io.airbyte.protocol.models.JsonSchemaReferenceTypes.TYPE_KEY; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.protocol.migrations.util.SchemaMigrations; -import io.airbyte.protocol.models.JsonSchemaReferenceTypes; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map.Entry; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.stream.StreamSupport; - -public class SchemaMigrationV1 { - - /** - * Perform the {type: foo} -> {$ref: foo} upgrade. Modifies the schema in-place. - */ - public static void upgradeSchema(final JsonNode schema) { - SchemaMigrations.mutateSchemas( - SchemaMigrationV1::isPrimitiveTypeDeclaration, - SchemaMigrationV1::upgradeTypeDeclaration, - schema); - } - - /** - * Perform the {$ref: foo} -> {type: foo} downgrade. Modifies the schema in-place. - */ - public static void downgradeSchema(final JsonNode schema) { - SchemaMigrations.mutateSchemas( - SchemaMigrationV1::isPrimitiveReferenceTypeDeclaration, - SchemaMigrationV1::downgradeTypeDeclaration, - schema); - } - - /** - * Detects any schema that looks like a primitive type declaration, e.g.: { "type": "string" } or { - * "type": ["string", "object"] } - */ - static boolean isPrimitiveTypeDeclaration(final JsonNode schema) { - if (!schema.isObject() || !schema.hasNonNull(TYPE_KEY)) { - return false; - } - final JsonNode typeNode = schema.get(TYPE_KEY); - if (typeNode.isArray()) { - return StreamSupport.stream(typeNode.spliterator(), false) - .anyMatch(n -> JsonSchemaReferenceTypes.PRIMITIVE_JSON_TYPES.contains(n.asText())); - } else { - return JsonSchemaReferenceTypes.PRIMITIVE_JSON_TYPES.contains(typeNode.asText()); - } - } - - /** - * Detects any schema that looks like a reference type declaration, e.g.: { "$ref": - * "WellKnownTypes.json...." } or { "oneOf": [{"$ref": "..."}, {"type": "object"}] } - */ - static boolean isPrimitiveReferenceTypeDeclaration(final JsonNode schema) { - if (!schema.isObject()) { - // Non-object schemas (i.e. true/false) never need to be modified - return false; - } else if (schema.hasNonNull(REF_KEY) && schema.get(REF_KEY).asText().startsWith("WellKnownTypes.json")) { - // If this schema has a $ref, then we need to convert it back to type/airbyte_type/format - return true; - } else if (schema.hasNonNull(ONEOF_KEY)) { - // If this is a oneOf with at least one primitive $ref option, then we should consider converting it - // back - final List subschemas = getSubschemas(schema, ONEOF_KEY); - return subschemas.stream().anyMatch( - subschema -> subschema.hasNonNull(REF_KEY) - && subschema.get(REF_KEY).asText().startsWith("WellKnownTypes.json")); - } else { - return false; - } - } - - /** - * Modifies the schema in-place to upgrade from the old-style type declaration to the new-style $ref - * declaration. Assumes that the schema is an ObjectNode containing a primitive declaration, i.e. - * either something like: {"type": "string"} or: {"type": ["string", "object"]} - *

- * In the latter case, the schema may contain subschemas. This method mutually recurses with - * {@link SchemaMigrations#mutateSchemas(Function, Consumer, JsonNode)} to upgrade those subschemas. - * - * @param schema An ObjectNode representing a primitive type declaration - */ - private static void upgradeTypeDeclaration(final JsonNode schema) { - final ObjectNode schemaNode = (ObjectNode) schema; - - if (schemaNode.hasNonNull("airbyte_type")) { - // If airbyte_type is defined, always respect it - final String referenceType = JsonSchemaReferenceTypes.LEGACY_AIRBYTE_PROPERY_TO_REFERENCE.get(schemaNode.get("airbyte_type").asText()); - schemaNode.removeAll(); - schemaNode.put(REF_KEY, referenceType); - } else { - // Otherwise, fall back to type/format - final JsonNode typeNode = schemaNode.get(TYPE_KEY); - if (typeNode.isTextual()) { - // If the type is a single string, then replace this node with the appropriate reference type - final String type = typeNode.asText(); - final String referenceType = getReferenceType(type, schemaNode); - schemaNode.removeAll(); - schemaNode.put(REF_KEY, referenceType); - } else { - // If type is an array of strings, then things are more complicated - final List types = StreamSupport.stream(typeNode.spliterator(), false) - .map(JsonNode::asText) - // Everything is implicitly nullable by just not declaring the `required `field - // so filter out any explicit null types - .filter(type -> !"null".equals(type)) - .toList(); - final boolean exactlyOneType = types.size() == 1; - if (exactlyOneType) { - // If there's only one type, e.g. {type: [string]}, just treat that as equivalent to {type: string} - final String type = types.get(0); - final String referenceType = getReferenceType(type, schemaNode); - schemaNode.removeAll(); - schemaNode.put(REF_KEY, referenceType); - } else { - // If there are multiple types, we'll need to convert this to a oneOf. - // For arrays and objects, we do a mutual recursion back into mutateSchemas to upgrade their - // subschemas. - final ArrayNode oneOfOptions = Jsons.arrayNode(); - for (final String type : types) { - final ObjectNode option = (ObjectNode) Jsons.emptyObject(); - switch (type) { - case "array" -> { - option.put(TYPE_KEY, "array"); - copyKey(schemaNode, option, "items"); - copyKey(schemaNode, option, "additionalItems"); - copyKey(schemaNode, option, "contains"); - upgradeSchema(option); - } - case "object" -> { - option.put(TYPE_KEY, "object"); - copyKey(schemaNode, option, "properties"); - copyKey(schemaNode, option, "patternProperties"); - copyKey(schemaNode, option, "additionalProperties"); - upgradeSchema(option); - } - default -> { - final String referenceType = getReferenceType(type, schemaNode); - option.put(REF_KEY, referenceType); - } - } - oneOfOptions.add(option); - } - schemaNode.removeAll(); - schemaNode.set(ONEOF_KEY, oneOfOptions); - } - } - } - } - - /** - * Modifies the schema in-place to downgrade from the new-style $ref declaration to the old-style - * type declaration. Assumes that the schema is an ObjectNode containing a primitive declaration, - * i.e. either something like: {"$ref": "WellKnownTypes..."} or: {"oneOf": [{"$ref": - * "WellKnownTypes..."}, ...]} - *

- * In the latter case, the schema may contain subschemas. This method mutually recurses with - * {@link SchemaMigrations#mutateSchemas(Function, Consumer, JsonNode)} to downgrade those - * subschemas. - * - * @param schema An ObjectNode representing a primitive type declaration - */ - private static void downgradeTypeDeclaration(final JsonNode schema) { - if (schema.hasNonNull(REF_KEY)) { - // If this is a direct type declaration, then we can just replace it with the old-style declaration - final String referenceType = schema.get(REF_KEY).asText(); - ((ObjectNode) schema).removeAll(); - ((ObjectNode) schema).setAll(JsonSchemaReferenceTypes.REFERENCE_TYPE_TO_OLD_TYPE.get(referenceType)); - } else if (schema.hasNonNull(ONEOF_KEY)) { - // If this is a oneOf, then we need to check whether we can recombine it into a single type - // declaration. - // This means we must do three things: - // 1. Downgrade each subschema - // 2. Build a new `type` array, containing the `type` of each subschema - // 3. Combine all the fields in each subschema (properties, items, etc) - // If any two subschemas have the same `type`, or the same field, then we can't combine them, but we - // should still downgrade them. - // See V0ToV1MigrationTest.CatalogDowngradeTest#testDowngradeMultiTypeFields for some examples. - - // We'll build up a node containing the combined subschemas. - final ObjectNode replacement = (ObjectNode) Jsons.emptyObject(); - // As part of this, we need to build up a list of `type` entries. For ease of access, we'll keep it - // in a List. - final List types = new ArrayList<>(); - - boolean canRecombineSubschemas = true; - for (final JsonNode subschemaNode : schema.get(ONEOF_KEY)) { - // No matter what - we always need to downgrade the subschema node. - downgradeSchema(subschemaNode); - - if (subschemaNode instanceof ObjectNode subschema) { - // If this subschema is an object, then we can attempt to combine it with the other subschemas. - - // First, update our list of types. - final JsonNode subschemaType = subschema.get(TYPE_KEY); - if (subschemaType != null) { - if (types.contains(subschemaType.asText())) { - // If another subschema has the same type, then we can't combine them. - canRecombineSubschemas = false; - } else { - types.add(subschemaType.asText()); - } - } - - // Then, update the combined schema with this subschema's fields. - if (canRecombineSubschemas) { - final Iterator> fields = subschema.fields(); - while (fields.hasNext()) { - final Entry field = fields.next(); - if (TYPE_KEY.equals(field.getKey())) { - // We're handling the `type` field outside this loop, so ignore it here. - continue; - } - if (replacement.has(field.getKey())) { - // A previous subschema is already using this field, so we should stop trying to combine them. - canRecombineSubschemas = false; - break; - } else { - replacement.set(field.getKey(), field.getValue()); - } - } - } - } else { - // If this subschema is a boolean, then the oneOf is doing something funky, and we shouldn't attempt - // to - // combine it into a single type entry - canRecombineSubschemas = false; - } - } - - if (canRecombineSubschemas) { - // Update our replacement node with the full list of types - final ArrayNode typeNode = Jsons.arrayNode(); - types.forEach(typeNode::add); - replacement.set(TYPE_KEY, typeNode); - - // And commit our changes to the actual schema node - ((ObjectNode) schema).removeAll(); - ((ObjectNode) schema).setAll(replacement); - } - } - } - - private static void copyKey(final ObjectNode source, final ObjectNode target, final String key) { - if (source.hasNonNull(key)) { - target.set(key, source.get(key)); - } - } - - /** - * Given a primitive (string/int/num/bool) type declaration _without_ an airbyte_type, get the - * appropriate $ref type. In most cases, this only depends on the "type" key. When type=string, also - * checks the "format" key. - */ - private static String getReferenceType(final String type, final ObjectNode schemaNode) { - return switch (type) { - case "string" -> { - if (schemaNode.hasNonNull("format")) { - yield switch (schemaNode.get("format").asText()) { - case "date" -> JsonSchemaReferenceTypes.DATE_REFERENCE; - // In these two cases, we default to the "with timezone" type, rather than "without timezone". - // This matches existing behavior in normalization. - case "date-time" -> JsonSchemaReferenceTypes.TIMESTAMP_WITH_TIMEZONE_REFERENCE; - case "time" -> JsonSchemaReferenceTypes.TIME_WITH_TIMEZONE_REFERENCE; - // If we don't recognize the format, just use a plain string - default -> JsonSchemaReferenceTypes.STRING_REFERENCE; - }; - } else if (schemaNode.hasNonNull("contentEncoding")) { - if ("base64".equals(schemaNode.get("contentEncoding").asText())) { - yield JsonSchemaReferenceTypes.BINARY_DATA_REFERENCE; - } else { - yield JsonSchemaReferenceTypes.STRING_REFERENCE; - } - } else { - yield JsonSchemaReferenceTypes.STRING_REFERENCE; - } - } - case "integer" -> JsonSchemaReferenceTypes.INTEGER_REFERENCE; - case "number" -> JsonSchemaReferenceTypes.NUMBER_REFERENCE; - case "boolean" -> JsonSchemaReferenceTypes.BOOLEAN_REFERENCE; - // This is impossible, because we'll only call this method on string/integer/number/boolean - default -> throw new IllegalStateException("Somehow got non-primitive type: " + type + " for schema: " + schemaNode); - }; - } - - private static List getSubschemas(final JsonNode schema, final String key) { - final List subschemas = new ArrayList<>(); - SchemaMigrations.findSubschemas(subschemas, schema, key); - return subschemas; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageDeserializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageDeserializer.java deleted file mode 100644 index e641a172e4c785..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageDeserializer.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.version.Version; - -public interface AirbyteMessageDeserializer { - - T deserialize(final JsonNode json); - - Version getTargetVersion(); - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericDeserializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericDeserializer.java deleted file mode 100644 index 8be50450793b5f..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericDeserializer.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.version.Version; -import lombok.Getter; - -public class AirbyteMessageGenericDeserializer implements AirbyteMessageDeserializer { - - @Getter - final Version targetVersion; - final Class typeClass; - - public AirbyteMessageGenericDeserializer(final Version targetVersion, final Class typeClass) { - this.targetVersion = targetVersion; - this.typeClass = typeClass; - } - - @Override - public T deserialize(JsonNode json) { - return Jsons.object(json, typeClass); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericSerializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericSerializer.java deleted file mode 100644 index ad43e35c1d146e..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericSerializer.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.version.Version; -import lombok.AllArgsConstructor; -import lombok.Getter; - -@AllArgsConstructor -public class AirbyteMessageGenericSerializer implements AirbyteMessageSerializer { - - @Getter - private final Version targetVersion; - - @Override - public String serialize(T message) { - return Jsons.serialize(message); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageSerializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageSerializer.java deleted file mode 100644 index 05b2b4e40834e0..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageSerializer.java +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import io.airbyte.commons.version.Version; - -public interface AirbyteMessageSerializer { - - String serialize(final T message); - - Version getTargetVersion(); - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java deleted file mode 100644 index a64b5201afbf9c..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.protocol.models.AirbyteMessage; - -public class AirbyteMessageV0Deserializer extends AirbyteMessageGenericDeserializer { - - public AirbyteMessageV0Deserializer() { - super(AirbyteProtocolVersion.V0, AirbyteMessage.class); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java deleted file mode 100644 index 2e28da22301831..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.protocol.models.AirbyteMessage; - -public class AirbyteMessageV0Serializer extends AirbyteMessageGenericSerializer { - - public AirbyteMessageV0Serializer() { - super(AirbyteProtocolVersion.V0); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Deserializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Deserializer.java deleted file mode 100644 index e85e637b73b3f1..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Deserializer.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.protocol.models.AirbyteMessage; - -public class AirbyteMessageV1Deserializer extends AirbyteMessageGenericDeserializer { - - public AirbyteMessageV1Deserializer() { - super(AirbyteProtocolVersion.V1, AirbyteMessage.class); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Serializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Serializer.java deleted file mode 100644 index caf2f0454de1ed..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Serializer.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import io.airbyte.commons.version.AirbyteProtocolVersion; -import io.airbyte.protocol.models.AirbyteMessage; - -public class AirbyteMessageV1Serializer extends AirbyteMessageGenericSerializer { - - public AirbyteMessageV1Serializer() { - super(AirbyteProtocolVersion.V1); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageMigratorTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageMigratorTest.java deleted file mode 100644 index 25ea40bd8b0353..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageMigratorTest.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import io.airbyte.commons.protocol.migrations.AirbyteMessageMigration; -import io.airbyte.commons.version.Version; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import java.util.List; -import java.util.Optional; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class AirbyteMessageMigratorTest { - - static final Version v0 = new Version("0.0.0"); - static final Version v1 = new Version("1.0.0"); - static final Version v2 = new Version("2.0.0"); - - record ObjectV0(String name0) {} - - record ObjectV1(String name1) {} - - record ObjectV2(String name2) {} - - static class Migrate0to1 implements AirbyteMessageMigration { - - @Override - public ObjectV0 downgrade(ObjectV1 message, Optional configuredAirbyteCatalog) { - return new ObjectV0(message.name1); - } - - @Override - public ObjectV1 upgrade(ObjectV0 message, Optional configuredAirbyteCatalog) { - return new ObjectV1(message.name0); - } - - @Override - public Version getPreviousVersion() { - return v0; - } - - @Override - public Version getCurrentVersion() { - return v1; - } - - } - - static class Migrate1to2 implements AirbyteMessageMigration { - - @Override - public ObjectV1 downgrade(ObjectV2 message, Optional configuredAirbyteCatalog) { - return new ObjectV1(message.name2); - } - - @Override - public ObjectV2 upgrade(ObjectV1 message, Optional configuredAirbyteCatalog) { - return new ObjectV2(message.name1); - } - - @Override - public Version getPreviousVersion() { - return v1; - } - - @Override - public Version getCurrentVersion() { - return v2; - } - - } - - AirbyteMessageMigrator migrator; - - @BeforeEach - void beforeEach() { - migrator = new AirbyteMessageMigrator( - List.of(new Migrate0to1(), new Migrate1to2())); - migrator.initialize(); - } - - @Test - void testDowngrade() { - final ObjectV2 obj = new ObjectV2("my name"); - - final ObjectV0 objDowngradedTo0 = migrator.downgrade(obj, v0, Optional.empty()); - assertEquals(obj.name2, objDowngradedTo0.name0); - - final ObjectV1 objDowngradedTo1 = migrator.downgrade(obj, v1, Optional.empty()); - assertEquals(obj.name2, objDowngradedTo1.name1); - - final ObjectV2 objDowngradedTo2 = migrator.downgrade(obj, v2, Optional.empty()); - assertEquals(obj.name2, objDowngradedTo2.name2); - } - - @Test - void testUpgrade() { - final ObjectV0 obj0 = new ObjectV0("my name 0"); - final ObjectV2 objUpgradedFrom0 = migrator.upgrade(obj0, v0, Optional.empty()); - assertEquals(obj0.name0, objUpgradedFrom0.name2); - - final ObjectV1 obj1 = new ObjectV1("my name 1"); - final ObjectV2 objUpgradedFrom1 = migrator.upgrade(obj1, v1, Optional.empty()); - assertEquals(obj1.name1, objUpgradedFrom1.name2); - - final ObjectV2 obj2 = new ObjectV2("my name 2"); - final ObjectV2 objUpgradedFrom2 = migrator.upgrade(obj2, v2, Optional.empty()); - assertEquals(obj2.name2, objUpgradedFrom2.name2); - } - - @Test - void testUnsupportedDowngradeShouldFailExplicitly() { - assertThrows(RuntimeException.class, () -> { - migrator.downgrade(new ObjectV2("woot"), new Version("5.0.0"), Optional.empty()); - }); - } - - @Test - void testUnsupportedUpgradeShouldFailExplicitly() { - assertThrows(RuntimeException.class, () -> { - migrator.upgrade(new ObjectV0("woot"), new Version("4.0.0"), Optional.empty()); - }); - } - - @Test - void testRegisterCollisionsShouldFail() { - assertThrows(RuntimeException.class, () -> { - migrator = new AirbyteMessageMigrator( - List.of(new Migrate0to1(), new Migrate1to2(), new Migrate0to1())); - migrator.initialize(); - }); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderTest.java deleted file mode 100644 index 2e0717a0176d69..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import io.airbyte.commons.protocol.serde.AirbyteMessageDeserializer; -import io.airbyte.commons.protocol.serde.AirbyteMessageSerializer; -import io.airbyte.commons.version.Version; -import java.util.Optional; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class AirbyteMessageSerDeProviderTest { - - AirbyteMessageSerDeProvider serDeProvider; - AirbyteMessageDeserializer deserV0; - AirbyteMessageDeserializer deserV1; - - AirbyteMessageSerializer serV0; - AirbyteMessageSerializer serV1; - - @BeforeEach - void beforeEach() { - serDeProvider = new AirbyteMessageSerDeProvider(); - - deserV0 = buildDeserializer(new Version("0.1.0")); - deserV1 = buildDeserializer(new Version("1.1.0")); - serDeProvider.registerDeserializer(deserV0); - serDeProvider.registerDeserializer(deserV1); - - serV0 = buildSerializer(new Version("0.2.0")); - serV1 = buildSerializer(new Version("1.0.0")); - serDeProvider.registerSerializer(serV0); - serDeProvider.registerSerializer(serV1); - } - - @Test - void testGetDeserializer() { - assertEquals(Optional.of(deserV0), serDeProvider.getDeserializer(new Version("0.1.0"))); - assertEquals(Optional.of(deserV0), serDeProvider.getDeserializer(new Version("0.2.0"))); - assertEquals(Optional.of(deserV1), serDeProvider.getDeserializer(new Version("1.1.0"))); - assertEquals(Optional.empty(), serDeProvider.getDeserializer(new Version("2.0.0"))); - } - - @Test - void testGetSerializer() { - assertEquals(Optional.of(serV0), serDeProvider.getSerializer(new Version("0.1.0"))); - assertEquals(Optional.of(serV1), serDeProvider.getSerializer(new Version("1.0.0"))); - assertEquals(Optional.empty(), serDeProvider.getSerializer(new Version("3.2.0"))); - } - - @Test - void testRegisterDeserializerShouldFailOnVersionCollision() { - AirbyteMessageDeserializer deser = buildDeserializer(new Version("0.2.0")); - assertThrows(RuntimeException.class, () -> { - serDeProvider.registerDeserializer(deser); - }); - } - - @Test - void testRegisterSerializerShouldFailOnVersionCollision() { - AirbyteMessageSerializer ser = buildSerializer(new Version("0.5.0")); - assertThrows(RuntimeException.class, () -> { - serDeProvider.registerSerializer(ser); - }); - } - - private AirbyteMessageDeserializer buildDeserializer(Version version) { - final AirbyteMessageDeserializer deser = mock(AirbyteMessageDeserializer.class); - when(deser.getTargetVersion()).thenReturn(version); - return deser; - } - - private AirbyteMessageSerializer buildSerializer(Version version) { - final AirbyteMessageSerializer ser = mock(AirbyteMessageSerializer.class); - when(ser.getTargetVersion()).thenReturn(version); - return ser; - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1Test.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1Test.java deleted file mode 100644 index 4d6905da984a49..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1Test.java +++ /dev/null @@ -1,1633 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations.v1; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.protocol.models.AirbyteCatalog; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.AirbyteMessage.Type; -import io.airbyte.protocol.models.AirbyteRecordMessage; -import io.airbyte.protocol.models.AirbyteStream; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; -import io.airbyte.validation.json.JsonSchemaValidator; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.List; -import java.util.Optional; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -// most of these tests rely on a doTest utility method for brevity, which hides the assertion. -@SuppressWarnings("PMD.JUnitTestsShouldIncludeAssert") -class AirbyteMessageMigrationV1Test { - - JsonSchemaValidator validator; - private AirbyteMessageMigrationV1 migration; - - @BeforeEach - void setup() throws URISyntaxException { - // TODO this should probably just get generated as part of the airbyte-protocol build, and - // airbyte-workers / airbyte-commons-protocol would reference it directly - final URI parentUri = MoreResources.readResourceAsFile("WellKnownTypes.json").getAbsoluteFile().toURI(); - validator = new JsonSchemaValidator(parentUri); - migration = new AirbyteMessageMigrationV1(validator); - } - - @Test - void testVersionMetadata() { - assertEquals("0.3.0", migration.getPreviousVersion().serialize()); - assertEquals("1.0.0", migration.getCurrentVersion().serialize()); - } - - @Nested - class CatalogUpgradeTest { - - @Test - void testBasicUpgrade() { - // This isn't actually a valid stream schema (since it's not an object) - // but this test case is mostly about preserving the message structure, so it's not super relevant - final JsonNode oldSchema = Jsons.deserialize( - """ - { - "type": "string" - } - """); - - final AirbyteMessage upgradedMessage = migration.upgrade(createCatalogMessage(oldSchema), Optional.empty()); - - final AirbyteMessage expectedMessage = Jsons.deserialize( - """ - { - "type": "CATALOG", - "catalog": { - "streams": [ - { - "json_schema": { - "$ref": "WellKnownTypes.json#/definitions/String" - } - } - ] - } - } - """, - AirbyteMessage.class); - assertEquals(expectedMessage, upgradedMessage); - } - - @Test - void testNullUpgrade() { - final io.airbyte.protocol.models.v0.AirbyteMessage oldMessage = new io.airbyte.protocol.models.v0.AirbyteMessage() - .withType(io.airbyte.protocol.models.v0.AirbyteMessage.Type.CATALOG); - final AirbyteMessage upgradedMessage = migration.upgrade(oldMessage, Optional.empty()); - final AirbyteMessage expectedMessage = new AirbyteMessage().withType(Type.CATALOG); - assertEquals(expectedMessage, upgradedMessage); - } - - /** - * Utility method to upgrade the oldSchema, and assert that the result is equal to expectedSchema - * - * @param oldSchemaString The schema to be upgraded - * @param expectedSchemaString The expected schema after upgrading - */ - private void doTest(final String oldSchemaString, final String expectedSchemaString) { - final JsonNode oldSchema = Jsons.deserialize(oldSchemaString); - - final AirbyteMessage upgradedMessage = migration.upgrade(createCatalogMessage(oldSchema), Optional.empty()); - - final JsonNode expectedSchema = Jsons.deserialize(expectedSchemaString); - assertEquals(expectedSchema, upgradedMessage.getCatalog().getStreams().get(0).getJsonSchema()); - } - - @Test - void testUpgradeAllPrimitives() { - doTest( - """ - { - "type": "object", - "properties": { - "example_string": { - "type": "string" - }, - "example_number": { - "type": "number" - }, - "example_integer": { - "type": "integer" - }, - "example_airbyte_integer": { - "type": "number", - "airbyte_type": "integer" - }, - "example_boolean": { - "type": "boolean" - }, - "example_timestamptz": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "example_timestamptz_implicit": { - "type": "string", - "format": "date-time" - }, - "example_timestamp_without_tz": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" - }, - "example_timez": { - "type": "string", - "format": "time", - "airbyte_type": "time_with_timezone" - }, - "example_timetz_implicit": { - "type": "string", - "format": "time" - }, - "example_time_without_tz": { - "type": "string", - "format": "time", - "airbyte_type": "time_without_timezone" - }, - "example_date": { - "type": "string", - "format": "date" - }, - "example_binary": { - "type": "string", - "contentEncoding": "base64" - } - } - } - """, - """ - { - "type": "object", - "properties": { - "example_string": { - "$ref": "WellKnownTypes.json#/definitions/String" - }, - "example_number": { - "$ref": "WellKnownTypes.json#/definitions/Number" - }, - "example_integer": { - "$ref": "WellKnownTypes.json#/definitions/Integer" - }, - "example_airbyte_integer": { - "$ref": "WellKnownTypes.json#/definitions/Integer" - }, - "example_boolean": { - "$ref": "WellKnownTypes.json#/definitions/Boolean" - }, - "example_timestamptz": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" - }, - "example_timestamptz_implicit": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" - }, - "example_timestamp_without_tz": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithoutTimezone" - }, - "example_timez": { - "$ref": "WellKnownTypes.json#/definitions/TimeWithTimezone" - }, - "example_timetz_implicit": { - "$ref": "WellKnownTypes.json#/definitions/TimeWithTimezone" - }, - "example_time_without_tz": { - "$ref": "WellKnownTypes.json#/definitions/TimeWithoutTimezone" - }, - "example_date": { - "$ref": "WellKnownTypes.json#/definitions/Date" - }, - "example_binary": { - "$ref": "WellKnownTypes.json#/definitions/BinaryData" - } - } - } - """); - } - - @Test - void testUpgradeNestedFields() { - doTest( - """ - { - "type": "object", - "properties": { - "basic_array": { - "items": {"type": "string"} - }, - "tuple_array": { - "items": [ - {"type": "string"}, - {"type": "integer"} - ], - "additionalItems": {"type": "string"}, - "contains": {"type": "integer"} - }, - "nested_object": { - "properties": { - "id": {"type": "integer"}, - "nested_oneof": { - "oneOf": [ - {"type": "string"}, - {"type": "integer"} - ] - }, - "nested_anyof": { - "anyOf": [ - {"type": "string"}, - {"type": "integer"} - ] - }, - "nested_allof": { - "allOf": [ - {"type": "string"}, - {"type": "integer"} - ] - }, - "nested_not": { - "not": [ - {"type": "string"}, - {"type": "integer"} - ] - } - }, - "patternProperties": { - "integer_.*": {"type": "integer"} - }, - "additionalProperties": {"type": "string"} - } - } - } - """, - """ - { - "type": "object", - "properties": { - "basic_array": { - "items": {"$ref": "WellKnownTypes.json#/definitions/String"} - }, - "tuple_array": { - "items": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ], - "additionalItems": {"$ref": "WellKnownTypes.json#/definitions/String"}, - "contains": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "nested_object": { - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "nested_oneof": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - }, - "nested_anyof": { - "anyOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - }, - "nested_allof": { - "allOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - }, - "nested_not": { - "not": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - } - }, - "patternProperties": { - "integer_.*": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "additionalProperties": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - } - } - """); - } - - @Test - void testUpgradeBooleanSchemas() { - // Most of these should never happen in reality, but let's handle them just in case - // The only ones that we're _really_ expecting are additionalItems and additionalProperties - final String schemaString = """ - { - "type": "object", - "properties": { - "basic_array": { - "items": true - }, - "tuple_array": { - "items": [true], - "additionalItems": true, - "contains": true - }, - "nested_object": { - "properties": { - "id": true, - "nested_oneof": { - "oneOf": [true] - }, - "nested_anyof": { - "anyOf": [true] - }, - "nested_allof": { - "allOf": [true] - }, - "nested_not": { - "not": [true] - } - }, - "patternProperties": { - "integer_.*": true - }, - "additionalProperties": true - } - } - } - """; - doTest(schemaString, schemaString); - } - - @Test - void testUpgradeEmptySchema() { - // Sources shouldn't do this, but we should have handling for it anyway, since it's not currently - // enforced by SATs - final String schemaString = """ - { - "type": "object", - "properties": { - "basic_array": { - "items": {} - }, - "tuple_array": { - "items": [{}], - "additionalItems": {}, - "contains": {} - }, - "nested_object": { - "properties": { - "id": {}, - "nested_oneof": { - "oneOf": [{}] - }, - "nested_anyof": { - "anyOf": [{}] - }, - "nested_allof": { - "allOf": [{}] - }, - "nested_not": { - "not": [{}] - } - }, - "patternProperties": { - "integer_.*": {} - }, - "additionalProperties": {} - } - } - } - """; - doTest(schemaString, schemaString); - } - - @Test - void testUpgradeLiteralSchema() { - // Verify that we do _not_ recurse into places we shouldn't - final String schemaString = """ - { - "type": "object", - "properties": { - "example_schema": { - "type": "object", - "default": {"type": "string"}, - "enum": [{"type": "string"}], - "const": {"type": "string"} - } - } - } - """; - doTest(schemaString, schemaString); - } - - @Test - void testUpgradeMalformedSchemas() { - // These schemas are "wrong" in some way. For example, normalization will currently treat - // bad_timestamptz as a string timestamp_with_timezone, - // i.e. it will disregard the option for a boolean. - // Generating this sort of schema is just wrong; sources shouldn't do this to begin with. But let's - // verify that we behave mostly correctly here. - doTest( - """ - { - "type": "object", - "properties": { - "bad_timestamptz": { - "type": ["boolean", "string"], - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "bad_integer": { - "type": "string", - "format": "date-time", - "airbyte_type": "integer" - } - } - } - """, - """ - { - "type": "object", - "properties": { - "bad_timestamptz": {"$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone"}, - "bad_integer": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - } - } - """); - } - - @Test - void testUpgradeMultiTypeFields() { - doTest( - """ - { - "type": "object", - "properties": { - "multityped_field": { - "type": ["string", "object", "array"], - "properties": { - "id": {"type": "string"} - }, - "patternProperties": { - "integer_.*": {"type": "integer"} - }, - "additionalProperties": {"type": "string"}, - "items": {"type": "string"}, - "additionalItems": {"type": "string"}, - "contains": {"type": "string"} - }, - "nullable_multityped_field": { - "type": ["null", "string", "array", "object"], - "items": [{"type": "string"}, {"type": "integer"}], - "properties": { - "id": {"type": "integer"} - } - }, - "multityped_date_field": { - "type": ["string", "integer"], - "format": "date" - }, - "sneaky_singletype_field": { - "type": ["string", "null"], - "format": "date-time" - } - } - } - """, - """ - { - "type": "object", - "properties": { - "multityped_field": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - { - "type": "object", - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/String"} - }, - "patternProperties": { - "integer_.*": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "additionalProperties": {"$ref": "WellKnownTypes.json#/definitions/String"} - }, - { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/String"}, - "additionalItems": {"$ref": "WellKnownTypes.json#/definitions/String"}, - "contains": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - ] - }, - "nullable_multityped_field": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - { - "type": "array", - "items": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - }, - { - "type": "object", - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - } - } - ] - }, - "multityped_date_field": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/Date"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - }, - "sneaky_singletype_field": {"$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone"} - } - } - """); - } - - private io.airbyte.protocol.models.v0.AirbyteMessage createCatalogMessage(final JsonNode schema) { - return new io.airbyte.protocol.models.v0.AirbyteMessage().withType(io.airbyte.protocol.models.v0.AirbyteMessage.Type.CATALOG) - .withCatalog( - new io.airbyte.protocol.models.v0.AirbyteCatalog().withStreams(List.of(new io.airbyte.protocol.models.v0.AirbyteStream().withJsonSchema( - schema)))); - } - - } - - @Nested - class RecordUpgradeTest { - - @Test - void testBasicUpgrade() { - final JsonNode oldData = Jsons.deserialize( - """ - { - "id": 42 - } - """); - - final AirbyteMessage upgradedMessage = migration.upgrade(createRecordMessage(oldData), Optional.empty()); - - final AirbyteMessage expectedMessage = Jsons.deserialize( - """ - { - "type": "RECORD", - "record": { - "data": { - "id": "42" - } - } - } - """, - AirbyteMessage.class); - assertEquals(expectedMessage, upgradedMessage); - } - - @Test - void testNullUpgrade() { - final io.airbyte.protocol.models.v0.AirbyteMessage oldMessage = new io.airbyte.protocol.models.v0.AirbyteMessage() - .withType(io.airbyte.protocol.models.v0.AirbyteMessage.Type.RECORD); - final AirbyteMessage upgradedMessage = migration.upgrade(oldMessage, Optional.empty()); - final AirbyteMessage expectedMessage = new AirbyteMessage().withType(Type.RECORD); - assertEquals(expectedMessage, upgradedMessage); - } - - /** - * Utility method to upgrade the oldData, and assert that the result is equal to expectedData - * - * @param oldDataString The data of the record to be upgraded - * @param expectedDataString The expected data after upgrading - */ - private void doTest(final String oldDataString, final String expectedDataString) { - final JsonNode oldData = Jsons.deserialize(oldDataString); - - final AirbyteMessage upgradedMessage = migration.upgrade(createRecordMessage(oldData), Optional.empty()); - - final JsonNode expectedData = Jsons.deserialize(expectedDataString); - assertEquals(expectedData, upgradedMessage.getRecord().getData()); - } - - @Test - void testNestedUpgrade() { - doTest( - """ - { - "int": 42, - "float": 42.0, - "float2": 42.2, - "sub_object": { - "sub_int": 42, - "sub_float": 42.0, - "sub_float2": 42.2 - }, - "sub_array": [42, 42.0, 42.2] - } - """, - """ - { - "int": "42", - "float": "42.0", - "float2": "42.2", - "sub_object": { - "sub_int": "42", - "sub_float": "42.0", - "sub_float2": "42.2" - }, - "sub_array": ["42", "42.0", "42.2"] - } - """); - } - - @Test - void testNonUpgradableValues() { - doTest( - """ - { - "boolean": true, - "string": "arst", - "sub_object": { - "boolean": true, - "string": "arst" - }, - "sub_array": [true, "arst"] - } - """, - """ - { - "boolean": true, - "string": "arst", - "sub_object": { - "boolean": true, - "string": "arst" - }, - "sub_array": [true, "arst"] - } - """); - } - - private io.airbyte.protocol.models.v0.AirbyteMessage createRecordMessage(final JsonNode data) { - return new io.airbyte.protocol.models.v0.AirbyteMessage().withType(io.airbyte.protocol.models.v0.AirbyteMessage.Type.RECORD) - .withRecord(new io.airbyte.protocol.models.v0.AirbyteRecordMessage().withData(data)); - } - - } - - @Nested - class CatalogDowngradeTest { - - @Test - void testBasicDowngrade() { - // This isn't actually a valid stream schema (since it's not an object) - // but this test case is mostly about preserving the message structure, so it's not super relevant - final JsonNode newSchema = Jsons.deserialize( - """ - { - "$ref": "WellKnownTypes.json#/definitions/String" - } - """); - - final io.airbyte.protocol.models.v0.AirbyteMessage downgradedMessage = migration.downgrade(createCatalogMessage(newSchema), Optional.empty()); - - final io.airbyte.protocol.models.v0.AirbyteMessage expectedMessage = Jsons.deserialize( - """ - { - "type": "CATALOG", - "catalog": { - "streams": [ - { - "json_schema": { - "type": "string" - } - } - ] - } - } - """, - io.airbyte.protocol.models.v0.AirbyteMessage.class); - assertEquals(expectedMessage, downgradedMessage); - } - - @Test - void testNullDowngrade() { - final AirbyteMessage oldMessage = new AirbyteMessage().withType(Type.CATALOG); - final io.airbyte.protocol.models.v0.AirbyteMessage upgradedMessage = migration.downgrade(oldMessage, Optional.empty()); - final io.airbyte.protocol.models.v0.AirbyteMessage expectedMessage = new io.airbyte.protocol.models.v0.AirbyteMessage() - .withType(io.airbyte.protocol.models.v0.AirbyteMessage.Type.CATALOG); - assertEquals(expectedMessage, upgradedMessage); - } - - /** - * Utility method to downgrade the oldSchema, and assert that the result is equal to expectedSchema - * - * @param oldSchemaString The schema to be downgraded - * @param expectedSchemaString The expected schema after downgrading - */ - private void doTest(final String oldSchemaString, final String expectedSchemaString) { - final JsonNode oldSchema = Jsons.deserialize(oldSchemaString); - - final io.airbyte.protocol.models.v0.AirbyteMessage downgradedMessage = migration.downgrade(createCatalogMessage(oldSchema), Optional.empty()); - - final JsonNode expectedSchema = Jsons.deserialize(expectedSchemaString); - assertEquals(expectedSchema, downgradedMessage.getCatalog().getStreams().get(0).getJsonSchema()); - } - - @Test - void testDowngradeAllPrimitives() { - doTest( - """ - { - "type": "object", - "properties": { - "example_string": { - "$ref": "WellKnownTypes.json#/definitions/String" - }, - "example_number": { - "$ref": "WellKnownTypes.json#/definitions/Number" - }, - "example_integer": { - "$ref": "WellKnownTypes.json#/definitions/Integer" - }, - "example_boolean": { - "$ref": "WellKnownTypes.json#/definitions/Boolean" - }, - "example_timestamptz": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" - }, - "example_timestamp_without_tz": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithoutTimezone" - }, - "example_timez": { - "$ref": "WellKnownTypes.json#/definitions/TimeWithTimezone" - }, - "example_time_without_tz": { - "$ref": "WellKnownTypes.json#/definitions/TimeWithoutTimezone" - }, - "example_date": { - "$ref": "WellKnownTypes.json#/definitions/Date" - }, - "example_binary": { - "$ref": "WellKnownTypes.json#/definitions/BinaryData" - } - } - } - """, - """ - { - "type": "object", - "properties": { - "example_string": { - "type": "string" - }, - "example_number": { - "type": "number" - }, - "example_integer": { - "type": "number", - "airbyte_type": "integer" - }, - "example_boolean": { - "type": "boolean" - }, - "example_timestamptz": { - "type": "string", - "airbyte_type": "timestamp_with_timezone", - "format": "date-time" - }, - "example_timestamp_without_tz": { - "type": "string", - "airbyte_type": "timestamp_without_timezone", - "format": "date-time" - }, - "example_timez": { - "type": "string", - "airbyte_type": "time_with_timezone", - "format": "time" - }, - "example_time_without_tz": { - "type": "string", - "airbyte_type": "time_without_timezone", - "format": "time" - }, - "example_date": { - "type": "string", - "format": "date" - }, - "example_binary": { - "type": "string", - "contentEncoding": "base64" - } - } - } - """); - } - - @Test - void testDowngradeNestedFields() { - doTest( - """ - { - "type": "object", - "properties": { - "basic_array": { - "items": {"$ref": "WellKnownTypes.json#/definitions/String"} - }, - "tuple_array": { - "items": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ], - "additionalItems": {"$ref": "WellKnownTypes.json#/definitions/String"}, - "contains": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "nested_object": { - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "nested_oneof": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone"} - ] - }, - "nested_anyof": { - "anyOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - }, - "nested_allof": { - "allOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - }, - "nested_not": { - "not": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - } - }, - "patternProperties": { - "integer_.*": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "additionalProperties": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - } - } - """, - """ - { - "type": "object", - "properties": { - "basic_array": { - "items": {"type": "string"} - }, - "tuple_array": { - "items": [ - {"type": "string"}, - {"type": "number", "airbyte_type": "integer"} - ], - "additionalItems": {"type": "string"}, - "contains": {"type": "number", "airbyte_type": "integer"} - }, - "nested_object": { - "properties": { - "id": {"type": "number", "airbyte_type": "integer"}, - "nested_oneof": { - "oneOf": [ - {"type": "string"}, - {"type": "string", "format": "date-time", "airbyte_type": "timestamp_with_timezone"} - ] - }, - "nested_anyof": { - "anyOf": [ - {"type": "string"}, - {"type": "number", "airbyte_type": "integer"} - ] - }, - "nested_allof": { - "allOf": [ - {"type": "string"}, - {"type": "number", "airbyte_type": "integer"} - ] - }, - "nested_not": { - "not": [ - {"type": "string"}, - {"type": "number", "airbyte_type": "integer"} - ] - } - }, - "patternProperties": { - "integer_.*": {"type": "number", "airbyte_type": "integer"} - }, - "additionalProperties": {"type": "string"} - } - } - } - """); - } - - @Test - void testDowngradeBooleanSchemas() { - // Most of these should never happen in reality, but let's handle them just in case - // The only ones that we're _really_ expecting are additionalItems and additionalProperties - final String schemaString = """ - { - "type": "object", - "properties": { - "basic_array": { - "items": true - }, - "tuple_array": { - "items": [true], - "additionalItems": true, - "contains": true - }, - "nested_object": { - "properties": { - "id": true, - "nested_oneof": { - "oneOf": [true] - }, - "nested_anyof": { - "anyOf": [true] - }, - "nested_allof": { - "allOf": [true] - }, - "nested_not": { - "not": [true] - } - }, - "patternProperties": { - "integer_.*": true - }, - "additionalProperties": true - } - } - } - """; - doTest(schemaString, schemaString); - } - - @Test - void testDowngradeEmptySchema() { - // Sources shouldn't do this, but we should have handling for it anyway, since it's not currently - // enforced by SATs - final String schemaString = """ - { - "type": "object", - "properties": { - "basic_array": { - "items": {} - }, - "tuple_array": { - "items": [{}], - "additionalItems": {}, - "contains": {} - }, - "nested_object": { - "properties": { - "id": {}, - "nested_oneof": { - "oneOf": [{}] - }, - "nested_anyof": { - "anyOf": [{}] - }, - "nested_allof": { - "allOf": [{}] - }, - "nested_not": { - "not": [{}] - } - }, - "patternProperties": { - "integer_.*": {} - }, - "additionalProperties": {} - } - } - } - """; - doTest(schemaString, schemaString); - } - - @Test - void testDowngradeLiteralSchema() { - // Verify that we do _not_ recurse into places we shouldn't - final String schemaString = """ - { - "type": "object", - "properties": { - "example_schema": { - "type": "object", - "default": {"$ref": "WellKnownTypes.json#/definitions/String"}, - "enum": [{"$ref": "WellKnownTypes.json#/definitions/String"}], - "const": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - } - } - """; - doTest(schemaString, schemaString); - } - - @Test - void testDowngradeMultiTypeFields() { - doTest( - """ - { - "type": "object", - "properties": { - "multityped_field": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - { - "type": "object", - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/String"} - }, - "patternProperties": { - "integer_.*": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "additionalProperties": {"$ref": "WellKnownTypes.json#/definitions/String"} - }, - { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/String"}, - "additionalItems": {"$ref": "WellKnownTypes.json#/definitions/String"}, - "contains": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - ] - }, - "multityped_date_field": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/Date"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - }, - "boolean_field": { - "oneOf": [ - true, - {"$ref": "WellKnownTypes.json#/definitions/String"}, - false - ] - }, - "conflicting_field": { - "oneOf": [ - {"type": "object", "properties": {"id": {"$ref": "WellKnownTypes.json#/definitions/String"}}}, - {"type": "object", "properties": {"name": {"$ref": "WellKnownTypes.json#/definitions/String"}}}, - {"$ref": "WellKnownTypes.json#/definitions/String"} - ] - }, - "conflicting_primitives": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/TimestampWithoutTimezone"}, - {"$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone"} - ] - } - } - } - """, - """ - { - "type": "object", - "properties": { - "multityped_field": { - "type": ["string", "object", "array"], - "properties": { - "id": {"type": "string"} - }, - "patternProperties": { - "integer_.*": {"type": "number", "airbyte_type": "integer"} - }, - "additionalProperties": {"type": "string"}, - "items": {"type": "string"}, - "additionalItems": {"type": "string"}, - "contains": {"type": "string"} - }, - "multityped_date_field": { - "type": ["string", "number"], - "format": "date", - "airbyte_type": "integer" - }, - "boolean_field": { - "oneOf": [ - true, - {"type": "string"}, - false - ] - }, - "conflicting_field": { - "oneOf": [ - {"type": "object", "properties": {"id": {"type": "string"}}}, - {"type": "object", "properties": {"name": {"type": "string"}}}, - {"type": "string"} - ] - }, - "conflicting_primitives": { - "oneOf": [ - {"type": "string", "format": "date-time", "airbyte_type": "timestamp_without_timezone"}, - {"type": "string", "format": "date-time", "airbyte_type": "timestamp_with_timezone"} - ] - } - } - } - """); - } - - @Test - void testDowngradeWeirdSchemas() { - // old_style_schema isn't actually valid (i.e. v1 schemas should always be using $ref) - // but we should check that it behaves well anyway - doTest( - """ - { - "type": "object", - "properties": { - "old_style_schema": {"type": "string"} - } - } - """, - """ - { - "type": "object", - "properties": { - "old_style_schema": {"type": "string"} - } - } - """); - } - - private AirbyteMessage createCatalogMessage(final JsonNode schema) { - return new AirbyteMessage().withType(AirbyteMessage.Type.CATALOG) - .withCatalog( - new AirbyteCatalog().withStreams(List.of(new AirbyteStream().withJsonSchema( - schema)))); - } - - } - - @Nested - class RecordDowngradeTest { - - private static final String STREAM_NAME = "foo_stream"; - private static final String NAMESPACE_NAME = "foo_namespace"; - - @Test - void testBasicDowngrade() { - final ConfiguredAirbyteCatalog catalog = createConfiguredAirbyteCatalog( - """ - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - """); - final JsonNode oldData = Jsons.deserialize( - """ - "42" - """); - - final io.airbyte.protocol.models.v0.AirbyteMessage downgradedMessage = new AirbyteMessageMigrationV1(validator) - .downgrade(createRecordMessage(oldData), Optional.of(catalog)); - - final io.airbyte.protocol.models.v0.AirbyteMessage expectedMessage = Jsons.deserialize( - """ - { - "type": "RECORD", - "record": { - "stream": "foo_stream", - "namespace": "foo_namespace", - "data": 42 - } - } - """, - io.airbyte.protocol.models.v0.AirbyteMessage.class); - assertEquals(expectedMessage, downgradedMessage); - } - - @Test - void testNullDowngrade() { - final AirbyteMessage oldMessage = new AirbyteMessage().withType(Type.RECORD); - final io.airbyte.protocol.models.v0.AirbyteMessage upgradedMessage = migration.downgrade(oldMessage, Optional.empty()); - final io.airbyte.protocol.models.v0.AirbyteMessage expectedMessage = new io.airbyte.protocol.models.v0.AirbyteMessage() - .withType(io.airbyte.protocol.models.v0.AirbyteMessage.Type.RECORD); - assertEquals(expectedMessage, upgradedMessage); - } - - /** - * Utility method to use the given catalog to downgrade the oldData, and assert that the result is - * equal to expectedDataString - * - * @param schemaString The JSON schema of the record - * @param oldDataString The data of the record to be downgraded - * @param expectedDataString The expected data after downgrading - */ - private void doTest(final String schemaString, final String oldDataString, final String expectedDataString) { - final ConfiguredAirbyteCatalog catalog = createConfiguredAirbyteCatalog(schemaString); - final JsonNode oldData = Jsons.deserialize(oldDataString); - - final io.airbyte.protocol.models.v0.AirbyteMessage downgradedMessage = new AirbyteMessageMigrationV1(validator) - .downgrade(createRecordMessage(oldData), Optional.of(catalog)); - - final JsonNode expectedDowngradedRecord = Jsons.deserialize(expectedDataString); - assertEquals(expectedDowngradedRecord, downgradedMessage.getRecord().getData()); - } - - @Test - void testNestedDowngrade() { - doTest( - """ - { - "type": "object", - "properties": { - "int": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "num": {"$ref": "WellKnownTypes.json#/definitions/Number"}, - "binary": {"$ref": "WellKnownTypes.json#/definitions/BinaryData"}, - "bool": {"$ref": "WellKnownTypes.json#/definitions/Boolean"}, - "object": { - "type": "object", - "properties": { - "int": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "arr": { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - } - } - }, - "array": { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "array_multitype": { - "type": "array", - "items": [{"$ref": "WellKnownTypes.json#/definitions/Integer"}, {"$ref": "WellKnownTypes.json#/definitions/String"}] - }, - "oneof": { - "type": "array", - "items": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - {"$ref": "WellKnownTypes.json#/definitions/Boolean"} - ] - } - } - } - } - """, - """ - { - "int": "42", - "num": "43.2", - "string": "42", - "bool": true, - "object": { - "int": "42" - }, - "array": ["42"], - "array_multitype": ["42", "42"], - "oneof": ["42", true], - "additionalProperty": "42" - } - """, - """ - { - "int": 42, - "num": 43.2, - "string": "42", - "bool": true, - "object": { - "int": 42 - }, - "array": [42], - "array_multitype": [42, "42"], - "oneof": [42, true], - "additionalProperty": "42" - } - """); - } - - @Test - void testWeirdDowngrade() { - doTest( - """ - { - "type": "object", - "properties": { - "raw_int": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "raw_num": {"$ref": "WellKnownTypes.json#/definitions/Number"}, - "bad_int": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "typeless_object": { - "properties": { - "foo": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - } - }, - "typeless_array": { - "items": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "arr_obj_union1": { - "type": ["array", "object"], - "items": { - "type": "object", - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "name": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - }, - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "name": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - }, - "arr_obj_union2": { - "type": ["array", "object"], - "items": { - "type": "object", - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "name": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - }, - "properties": { - "id": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "name": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - }, - "empty_oneof": { - "oneOf": [] - } - } - } - """, - """ - { - "raw_int": 42, - "raw_num": 43.2, - "bad_int": "foo", - "typeless_object": { - "foo": "42" - }, - "typeless_array": ["42"], - "arr_obj_union1": [{"id": "42", "name": "arst"}, {"id": "43", "name": "qwfp"}], - "arr_obj_union2": {"id": "42", "name": "arst"}, - "empty_oneof": "42" - } - """, - """ - { - "raw_int": 42, - "raw_num": 43.2, - "bad_int": "foo", - "typeless_object": { - "foo": 42 - }, - "typeless_array": [42], - "arr_obj_union1": [{"id": 42, "name": "arst"}, {"id": 43, "name": "qwfp"}], - "arr_obj_union2": {"id": 42, "name": "arst"}, - "empty_oneof": "42" - } - """); - } - - @Test - void testEmptySchema() { - doTest( - """ - { - "type": "object", - "properties": { - "empty_schema_primitive": {}, - "empty_schema_array": {}, - "empty_schema_object": {}, - "implicit_array": { - "items": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "implicit_object": { - "properties": { - "foo": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - } - } - } - } - """, - """ - { - "empty_schema_primitive": "42", - "empty_schema_array": ["42", false], - "empty_schema_object": {"foo": "42"}, - "implicit_array": ["42"], - "implicit_object": {"foo": "42"} - } - """, - """ - { - "empty_schema_primitive": "42", - "empty_schema_array": ["42", false], - "empty_schema_object": {"foo": "42"}, - "implicit_array": [42], - "implicit_object": {"foo": 42} - } - """); - } - - @Test - void testBacktracking() { - // These test cases verify that we correctly choose the most-correct oneOf option. - doTest( - """ - { - "type": "object", - "properties": { - "valid_option": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/Boolean"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - {"$ref": "WellKnownTypes.json#/definitions/String"} - ] - }, - "all_invalid": { - "oneOf": [ - { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/Boolean"} - } - ] - }, - "nested_oneof": { - "oneOf": [ - { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - { - "type": "array", - "items": { - "type": "object", - "properties": { - "foo": { - "oneOf": [ - {"$ref": "WellKnownTypes.json#/definitions/Boolean"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - } - } - } - } - ] - }, - "mismatched_primitive": { - "oneOf": [ - { - "type": "object", - "properties": { - "foo": {"type": "object"}, - "bar": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - }, - { - "type": "object", - "properties": { - "foo": {"$ref": "WellKnownTypes.json#/definitions/Boolean"}, - "bar": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - } - } - ] - }, - "mismatched_text": { - "oneOf": [ - { - "type": "object", - "properties": { - "foo": {"type": "object"}, - "bar": {"$ref": "WellKnownTypes.json#/definitions/String"} - } - }, - { - "type": "object", - "properties": { - "foo": {"$ref": "WellKnownTypes.json#/definitions/String"}, - "bar": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - } - } - ] - }, - "mismatch_array": { - "oneOf": [ - { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - { - "type": "array", - "items": [ - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/String"}, - {"$ref": "WellKnownTypes.json#/definitions/Integer"} - ] - } - ] - } - } - } - """, - """ - { - "valid_option": "42", - "all_invalid": ["42", "arst"], - "nested_oneof": [{"foo": "42"}], - "mismatched_primitive": { - "foo": true, - "bar": "42" - }, - "mismatched_text": { - "foo": "bar", - "bar": "42" - }, - "mismatch_array": ["arst", "41", "42"] - } - """, - """ - { - "valid_option": 42, - "all_invalid": [42, "arst"], - "nested_oneof": [{"foo": 42}], - "mismatched_primitive": { - "foo": true, - "bar": 42 - }, - "mismatched_text": { - "foo": "bar", - "bar": 42 - }, - "mismatch_array": ["arst", "41", 42] - } - """); - } - - @Test - void testIncorrectSchema() { - doTest( - """ - { - "type": "object", - "properties": { - "bad_int": {"$ref": "WellKnownTypes.json#/definitions/Integer"}, - "bad_int_array": { - "type": "array", - "items": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - }, - "bad_int_obj": { - "type": "object", - "properties": { - "foo": {"$ref": "WellKnownTypes.json#/definitions/Integer"} - } - } - } - } - """, - """ - { - "bad_int": "arst", - "bad_int_array": ["arst"], - "bad_int_obj": {"foo": "arst"} - } - """, - """ - { - "bad_int": "arst", - "bad_int_array": ["arst"], - "bad_int_obj": {"foo": "arst"} - } - """); - } - - private ConfiguredAirbyteCatalog createConfiguredAirbyteCatalog(final String schema) { - return new ConfiguredAirbyteCatalog() - .withStreams(List.of(new ConfiguredAirbyteStream().withStream(new io.airbyte.protocol.models.AirbyteStream() - .withName(STREAM_NAME) - .withNamespace(NAMESPACE_NAME) - .withJsonSchema(Jsons.deserialize(schema))))); - } - - private AirbyteMessage createRecordMessage(final JsonNode data) { - return new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(STREAM_NAME).withNamespace(NAMESPACE_NAME).withData(data)); - } - - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1Test.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1Test.java deleted file mode 100644 index 1a663ec2cffec9..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1Test.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.migrations.v1; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.AirbyteStream; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; -import java.util.List; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -/** - * These depend on the same {@link SchemaMigrationV1} class as - * {@link io.airbyte.commons.protocol.migrations.v1.AirbyteMessageMigrationV1}. So, uh, I didn't - * bother writing a ton of tests for it. - * - * Check out {@link AirbyteMessageMigrationV1} for more comprehensive tests. Theoretically - * SchemaMigrationV1 should have its own set of tests, but for various (development history-related) - * reasons, that would be a lot of work. - */ -class ConfiguredAirbyteCatalogMigrationV1Test { - - private ConfiguredAirbyteCatalogMigrationV1 migration; - - @BeforeEach - void setup() { - migration = new ConfiguredAirbyteCatalogMigrationV1(); - } - - @Test - void testVersionMetadata() { - assertEquals("0.3.0", migration.getPreviousVersion().serialize()); - assertEquals("1.0.0", migration.getCurrentVersion().serialize()); - } - - @Test - void testBasicUpgrade() { - // This isn't actually a valid stream schema (since it's not an object) - // but this test case is mostly about preserving the message structure, so it's not super relevant - final io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog downgradedCatalog = new io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog() - .withStreams(List.of( - new io.airbyte.protocol.models.v0.ConfiguredAirbyteStream().withStream(new io.airbyte.protocol.models.v0.AirbyteStream().withJsonSchema( - Jsons.deserialize( - """ - { - "type": "string" - } - """))))); - - final ConfiguredAirbyteCatalog upgradedMessage = migration.upgrade(downgradedCatalog); - - final ConfiguredAirbyteCatalog expectedMessage = Jsons.deserialize( - """ - { - "streams": [ - { - "stream": { - "json_schema": { - "$ref": "WellKnownTypes.json#/definitions/String" - } - } - } - ] - } - """, - ConfiguredAirbyteCatalog.class); - assertEquals(expectedMessage, upgradedMessage); - } - - @Test - void testBasicDowngrade() { - // This isn't actually a valid stream schema (since it's not an object) - // but this test case is mostly about preserving the message structure, so it's not super relevant - final ConfiguredAirbyteCatalog upgradedCatalog = new ConfiguredAirbyteCatalog() - .withStreams(List.of( - new ConfiguredAirbyteStream().withStream(new AirbyteStream().withJsonSchema( - Jsons.deserialize(""" - { - "$ref": "WellKnownTypes.json#/definitions/String" - } - """))))); - - final io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog downgradedMessage = migration.downgrade(upgradedCatalog); - - final io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog expectedMessage = Jsons.deserialize( - """ - { - "streams": [ - { - "stream": { - "json_schema": { - "type": "string" - } - } - } - ] - } - """, - io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog.class); - assertEquals(expectedMessage, downgradedMessage); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0SerDeTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0SerDeTest.java deleted file mode 100644 index 80a398497ded10..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0SerDeTest.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.AirbyteMessage.Type; -import io.airbyte.protocol.models.ConnectorSpecification; -import java.net.URI; -import java.net.URISyntaxException; -import org.junit.jupiter.api.Test; - -class AirbyteMessageV0SerDeTest { - - @Test - void v0SerDeRoundTripTest() throws URISyntaxException { - final AirbyteMessageV0Deserializer deser = new AirbyteMessageV0Deserializer(); - final AirbyteMessageV0Serializer ser = new AirbyteMessageV0Serializer(); - - final AirbyteMessage message = new AirbyteMessage() - .withType(Type.SPEC) - .withSpec( - new ConnectorSpecification() - .withProtocolVersion("0.3.0") - .withDocumentationUrl(new URI("file:///tmp/doc"))); - - final String serializedMessage = ser.serialize(message); - final AirbyteMessage deserializedMessage = deser.deserialize(Jsons.deserialize(serializedMessage)); - - assertEquals(message, deserializedMessage); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1SerDeTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1SerDeTest.java deleted file mode 100644 index 1a1e186c515f77..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1SerDeTest.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.protocol.serde; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.AirbyteMessage.Type; -import io.airbyte.protocol.models.ConnectorSpecification; -import java.net.URI; -import java.net.URISyntaxException; -import org.junit.jupiter.api.Test; - -class AirbyteMessageV1SerDeTest { - - @Test - void v1SerDeRoundTripTest() throws URISyntaxException { - final AirbyteMessageV1Deserializer deser = new AirbyteMessageV1Deserializer(); - final AirbyteMessageV1Serializer ser = new AirbyteMessageV1Serializer(); - - final AirbyteMessage message = new AirbyteMessage() - .withType(Type.SPEC) - .withSpec( - new ConnectorSpecification() - .withProtocolVersion("1.0.0") - .withDocumentationUrl(new URI("file:///tmp/doc"))); - - final String serializedMessage = ser.serialize(message); - final AirbyteMessage deserializedMessage = deser.deserialize(Jsons.deserialize(serializedMessage)); - - assertEquals(message, deserializedMessage); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/resources/WellKnownTypes.json b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/resources/WellKnownTypes.json deleted file mode 100644 index 9e4d6656deae7a..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/resources/WellKnownTypes.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "definitions": { - "String": { - "type": "string", - "description": "Arbitrary text" - }, - "BinaryData": { - "type": "string", - "description": "Arbitrary binary data. Represented as base64-encoded strings in the JSON transport. In the future, if we support other transports, may be encoded differently.\n", - "pattern": "^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" - }, - "Date": { - "type": "string", - "oneOf": [ - { - "pattern": "^\\d{4}-\\d{2}-\\d{2}( BC)?$" - }, - { - "enum": ["Infinity", "-Infinity"] - } - ], - "description": "RFC 3339\u00a75.6's full-date format, extended with BC era support and (-)Infinity" - }, - "TimestampWithTimezone": { - "type": "string", - "oneOf": [ - { - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|[+\\-]\\d{1,2}:\\d{2})( BC)?$" - }, - { - "enum": ["Infinity", "-Infinity"] - } - ], - "description": "An instant in time. Frequently simply referred to as just a timestamp, or timestamptz. Uses RFC 3339\u00a75.6's date-time format, requiring a \"T\" separator, and extended with BC era support and (-)Infinity. Note that we do _not_ accept Unix epochs here.\n" - }, - "TimestampWithoutTimezone": { - "type": "string", - "oneOf": [ - { - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?( BC)?$" - }, - { - "enum": ["Infinity", "-Infinity"] - } - ], - "description": "Also known as a localdatetime, or just datetime. Under RFC 3339\u00a75.6, this would be represented as `full-date \"T\" partial-time`, extended with BC era support and (-)Infinity.\n" - }, - "TimeWithTimezone": { - "type": "string", - "pattern": "^\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|[+\\-]\\d{1,2}:\\d{2})$", - "description": "An RFC 3339\u00a75.6 full-time" - }, - "TimeWithoutTimezone": { - "type": "string", - "pattern": "^\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?$", - "description": "An RFC 3339\u00a75.6 partial-time" - }, - "Number": { - "type": "string", - "oneOf": [ - { - "pattern": "-?(0|[0-9]\\d*)(\\.\\d+)?" - }, - { - "enum": ["Infinity", "-Infinity", "NaN"] - } - ], - "description": "Note the mix of regex validation for normal numbers, and enum validation for special values." - }, - "Integer": { - "type": "string", - "oneOf": [ - { - "pattern": "-?(0|[0-9]\\d*)" - }, - { - "enum": ["Infinity", "-Infinity", "NaN"] - } - ] - }, - "Boolean": { - "type": "boolean", - "description": "Note the direct usage of a primitive boolean rather than string. Unlike Numbers and Integers, we don't expect unusual values here." - } - } -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/LICENSE b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/LICENSE deleted file mode 100644 index ec45d182fcb901..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2020 Airbyte, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle deleted file mode 100644 index c92e3bdac1ecbe..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle +++ /dev/null @@ -1,19 +0,0 @@ -plugins { - id 'java-library' -} - -java { - compileJava { - options.compilerArgs += "-Xlint:-varargs,-try,-deprecation" - } - compileTestJava { - options.compilerArgs += "-Xlint:-try" - } -} - -dependencies { - // Dependencies for this module should be specified in the top-level build.gradle. See readme for more explanation. - - // this dependency is an exception to the above rule because it is only used INTERNALLY to the commons library. - implementation 'com.jayway.jsonpath:json-path:2.7.0' -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/readme.md b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/readme.md deleted file mode 100644 index 1f953216d630e2..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/readme.md +++ /dev/null @@ -1,11 +0,0 @@ -# airbyte-commons - -Common java helpers. - -This submodule is inherited by all other java modules in the monorepo! It is therefore important that we do not add dependencies to it, as those -dependencies will also be added to every java module. The only dependencies that this module uses are the ones declared in the `build.gradle` at the -root of the Airbyte monorepo. In other words it only uses dependencies that are already shared across all modules. The `dependencies` section of -the `build.gradle` of `airbyte-commons` should always be empty. - -For other common java code that needs to be shared across modules that requires additional dependencies, we follow this -convention: `airbyte-commons-`. See for example `airbyte-commons-cli`. diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/LICENSE b/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/LICENSE deleted file mode 100644 index ec45d182fcb901..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2020 Airbyte, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/build.gradle deleted file mode 100644 index 84674554e773b8..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/build.gradle +++ /dev/null @@ -1,10 +0,0 @@ -plugins { - id "java-library" -} - -dependencies { - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - implementation 'com.networknt:json-schema-validator:1.0.72' - // needed so that we can follow $ref when parsing json. jackson does not support this natively. - implementation 'me.andrz.jackson:jackson-json-reference-core:0.3.2' -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/readme.md b/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/readme.md deleted file mode 100644 index 57eff20af41c18..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/readme.md +++ /dev/null @@ -1,7 +0,0 @@ -# airbyte-json-validation - -This module contains shared Java code for validating JSON objects. - -## Key Files -* `JsonSchemaValidator.java` is the main entrypoint into this library, defining convenience methods for validation. -* `ConfigSchemaValidator.java` is additional sugar to make it easy to validate objects whose schemas are defined in `ConfigSchema`. diff --git a/airbyte-cdk/java/airbyte-cdk/azure-blob-storage-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/azure-blob-storage-destinations/build.gradle new file mode 100644 index 00000000000000..4d1b516a62cfc3 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/azure-blob-storage-destinations/build.gradle @@ -0,0 +1,7 @@ +dependencies { + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + implementation project(':airbyte-cdk:java:airbyte-cdk:core') + + implementation 'com.azure:azure-storage-blob:12.12.0' +} + diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageConfig.java b/airbyte-cdk/java/airbyte-cdk/azure-blob-storage-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageConfig.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageConfig.java rename to airbyte-cdk/java/airbyte-cdk/azure-blob-storage-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageConfig.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopier.java b/airbyte-cdk/java/airbyte-cdk/azure-blob-storage-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopier.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopier.java rename to airbyte-cdk/java/airbyte-cdk/azure-blob-storage-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopier.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopierFactory.java b/airbyte-cdk/java/airbyte-cdk/azure-blob-storage-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopierFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopierFactory.java rename to airbyte-cdk/java/airbyte-cdk/azure-blob-storage-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopierFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/README.md b/airbyte-cdk/java/airbyte-cdk/config-models-oss/README.md deleted file mode 100644 index 996bcf073ec852..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/config-models-oss/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Config Models - -This module uses `jsonschema2pojo` to generate Java config objects from [json schema](https://json-schema.org/) definitions. See [build.gradle](./build.gradle) for details. - -## How to use -- Update json schema under: - ``` - src/main/resources/types/ - ``` -- Run the following command under the project root: - ```sh - ./gradlew airbyte-cdk:java:airbyte-cdk:config-models-oss:generateJsonSchema2Pojo - ``` - The generated file is under: - ``` - build/generated/src/gen/java/io/airbyte/config/ - ``` - -## Reference -- [`jsonschema2pojo` plugin](https://github.com/joelittlejohn/jsonschema2pojo/tree/master/jsonschema2pojo-gradle-plugin). diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/build.gradle b/airbyte-cdk/java/airbyte-cdk/config-models-oss/build.gradle deleted file mode 100644 index f9146068689d8b..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/config-models-oss/build.gradle +++ /dev/null @@ -1,36 +0,0 @@ -import org.jsonschema2pojo.SourceType - -plugins { - id "java-library" - id "com.github.eirnym.js2p" version "1.0" -} - -java { - compileJava { - options.compilerArgs += "-Xlint:-unchecked" - } -} - -dependencies { - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') -} - -jsonSchema2Pojo { - sourceType = SourceType.YAMLSCHEMA - source = files("${sourceSets.main.output.resourcesDir}/types") - targetDirectory = new File(project.buildDir, 'generated/src/gen/java/') - - targetPackage = 'io.airbyte.configoss' - useLongIntegers = true - - removeOldOutput = true - - generateBuilders = true - includeConstructors = false - includeSetters = true - serializable = true -} -tasks.register('generate').configure { - dependsOn tasks.named('generateJsonSchema2Pojo') -} diff --git a/airbyte-cdk/java/airbyte-cdk/core/build.gradle b/airbyte-cdk/java/airbyte-cdk/core/build.gradle index 8b35718ef55d18..5a955f720e2819 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/core/build.gradle @@ -1,105 +1,48 @@ - java { + // TODO: rewrite code to avoid javac wornings in the first place compileJava { - options.compilerArgs += "-Xlint:-deprecation,-try,-rawtypes,-overloads,-cast,-unchecked" + options.compilerArgs += "-Xlint:-deprecation,-try,-rawtypes,-overloads" } compileTestJava { options.compilerArgs += "-Xlint:-try,-divzero,-cast" } -} - -configurations.all { - resolutionStrategy { - // TODO: Diagnose conflicting dependencies and remove these force overrides: - force 'org.mockito:mockito-core:4.6.1' + compileTestFixturesJava { + options.compilerArgs += "-Xlint:-cast,-deprecation" } } dependencies { - // Exported dependencies from upstream projects - api libs.airbyte.protocol - api libs.hikaricp - api libs.jooq - api libs.jooq.meta - - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-api') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - testCompileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') - testImplementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - - // SSH dependencies - implementation 'net.i2p.crypto:eddsa:0.3.0' - - // First party test dependencies - testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:db-sources')) + api 'com.datadoghq:dd-trace-api:1.28.0' + api 'com.datadoghq:dd-trace-ot:1.28.0' + api 'com.zaxxer:HikariCP:5.1.0' + api 'org.jooq:jooq:3.16.23' + api 'org.apache.commons:commons-csv:1.10.0' - testFixturesImplementation "org.hamcrest:hamcrest-all:1.3" + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') - testImplementation libs.bundles.junit - testImplementation libs.junit.jupiter.api - testImplementation libs.junit.jupiter.params - testImplementation 'org.junit.platform:junit-platform-launcher:1.7.0' - testImplementation libs.junit.jupiter.engine - implementation libs.jooq - implementation 'net.sourceforge.argparse4j:argparse4j:0.8.1' - implementation "io.aesy:datasize:1.0.0" - implementation libs.apache.commons - implementation libs.apache.commons.lang - testImplementation 'commons-lang:commons-lang:2.6' implementation 'commons-cli:commons-cli:1.4' - implementation 'org.apache.commons:commons-csv:1.4' - - // Optional dependencies - // TODO: Change these to 'compileOnly' or 'testCompileOnly' - implementation 'com.azure:azure-storage-blob:12.12.0' - implementation('com.google.cloud:google-cloud-bigquery:1.133.1') - implementation 'org.mongodb:mongodb-driver-sync:4.3.0' - implementation libs.postgresql - - // testImplementation libs.junit.jupiter.api - implementation libs.hikaricp - implementation libs.debezium.api - implementation libs.debezium.embedded - implementation libs.debezium.mysql - implementation libs.debezium.postgres - implementation libs.debezium.mongodb - - api libs.bundles.datadog + implementation 'io.aesy:datasize:1.0.0' + implementation 'net.i2p.crypto:eddsa:0.3.0' + implementation 'org.apache.httpcomponents:httpcore:4.4.16' + implementation 'org.apache.logging.log4j:log4j-layout-template-json:2.17.2' implementation 'org.apache.sshd:sshd-mina:2.11.0' - - implementation libs.testcontainers - implementation libs.testcontainers.mysql - implementation libs.testcontainers.jdbc - implementation libs.testcontainers.postgresql - testImplementation libs.testcontainers.jdbc - testImplementation libs.testcontainers.mysql - testImplementation libs.testcontainers.postgresql - implementation 'org.codehaus.plexus:plexus-utils:3.4.2' - // bouncycastle is pinned to version-match the transitive dependency from kubernetes client-java // because a version conflict causes "parameter object not a ECParameterSpec" on ssh tunnel initiation implementation 'org.bouncycastle:bcpkix-jdk15on:1.66' implementation 'org.bouncycastle:bcprov-jdk15on:1.66' implementation 'org.bouncycastle:bctls-jdk15on:1.66' - // Lombok - implementation 'org.projectlombok:lombok:1.18.20' - annotationProcessor 'org.projectlombok:lombok:1.18.20' - testFixturesImplementation 'org.projectlombok:lombok:1.18.20' - testFixturesAnnotationProcessor 'org.projectlombok:lombok:1.18.20' + testFixturesApi 'org.testcontainers:testcontainers:1.19.0' + testFixturesApi 'org.testcontainers:jdbc:1.19.0' - testImplementation libs.junit.jupiter.system.stubs - - implementation libs.jackson.annotations - implementation group: 'org.apache.logging.log4j', name: 'log4j-layout-template-json', version: '2.17.2' + testImplementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:db-sources')) + testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:datastore-postgres')) - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation 'org.xerial.snappy:snappy-java:1.1.8.4' - testImplementation 'org.mockito:mockito-core:4.6.1' + testImplementation 'mysql:mysql-connector-java:8.0.33' + testImplementation 'org.postgresql:postgresql:42.6.0' + testImplementation 'org.testcontainers:mysql:1.19.0' + testImplementation 'org.testcontainers:postgresql:1.19.0' + testImplementation 'org.xbib.elasticsearch:joptsimple:6.3.2.1' } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java index e7b8514cb7f518..c87c8047501ca1 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java @@ -28,8 +28,8 @@ import java.time.OffsetTime; import java.time.chrono.IsoEra; import java.time.format.DateTimeParseException; +import java.util.Base64; import java.util.Collections; -import javax.xml.bind.DatatypeConverter; /** * Source operation skeleton for JDBC compatible databases. @@ -222,7 +222,7 @@ protected void setString(final PreparedStatement preparedStatement, final int pa } protected void setBinary(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { - preparedStatement.setBytes(parameterIndex, DatatypeConverter.parseBase64Binary(value)); + preparedStatement.setBytes(parameterIndex, Base64.getDecoder().decode(value)); } protected ObjectType getObject(final ResultSet resultSet, final int index, final Class clazz) throws SQLException { diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/JdbcConnector.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/JdbcConnector.java index 1037c91858ec73..44f749e1694ab9 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/JdbcConnector.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/JdbcConnector.java @@ -4,8 +4,6 @@ package io.airbyte.cdk.integrations; -import static org.postgresql.PGProperty.CONNECT_TIMEOUT; - import io.airbyte.cdk.db.factory.DatabaseDriver; import java.time.Duration; import java.time.temporal.ChronoUnit; @@ -15,6 +13,9 @@ public abstract class JdbcConnector extends BaseConnector { + public static final String POSTGRES_CONNECT_TIMEOUT_KEY = "connectTimeout"; + public static final Duration POSTGRES_CONNECT_TIMEOUT_DEFAULT_DURATION = Duration.ofSeconds(10); + public static final String CONNECT_TIMEOUT_KEY = "connectTimeout"; public static final Duration CONNECT_TIMEOUT_DEFAULT = Duration.ofSeconds(60); @@ -44,8 +45,8 @@ protected Duration getConnectionTimeout(final Map connectionProp */ public static Duration getConnectionTimeout(final Map connectionProperties, String driverClassName) { final Optional parsedConnectionTimeout = switch (DatabaseDriver.findByDriverClassName(driverClassName)) { - case POSTGRESQL -> maybeParseDuration(connectionProperties.get(CONNECT_TIMEOUT.getName()), ChronoUnit.SECONDS) - .or(() -> maybeParseDuration(CONNECT_TIMEOUT.getDefaultValue(), ChronoUnit.SECONDS)); + case POSTGRESQL -> maybeParseDuration(connectionProperties.get(POSTGRES_CONNECT_TIMEOUT_KEY), ChronoUnit.SECONDS) + .or(() -> Optional.of(POSTGRES_CONNECT_TIMEOUT_DEFAULT_DURATION)); case MYSQL -> maybeParseDuration(connectionProperties.get("connectTimeout"), ChronoUnit.MILLIS); case MSSQLSERVER -> maybeParseDuration(connectionProperties.get("loginTimeout"), ChronoUnit.SECONDS); default -> maybeParseDuration(connectionProperties.get(CONNECT_TIMEOUT_KEY), ChronoUnit.SECONDS) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java index 56eecef479ef76..94c75b6f011801 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java @@ -12,8 +12,8 @@ import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.validation.constraints.NotNull; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshHelpers.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshHelpers.java index 367d429bdc2a8c..2462cc939d986e 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshHelpers.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshHelpers.java @@ -10,8 +10,6 @@ import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.io.IOException; import java.util.Optional; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.testcontainers.containers.Container; public class SshHelpers { @@ -39,31 +37,4 @@ public static ConnectorSpecification injectSshIntoSpec(final ConnectorSpecificat propNode.set("tunnel_method", tunnelMethod); return originalSpec; } - - /** - * Returns the inner docker network ip address and port of a container. This can be used to reach a - * container from another container running on the same network - * - * @param container container - * @return a pair of host and port - */ - public static ImmutablePair getInnerContainerAddress(final Container container) { - return ImmutablePair.of( - container.getContainerInfo().getNetworkSettings().getNetworks().entrySet().stream().findFirst().get().getValue().getIpAddress(), - (Integer) container.getExposedPorts().stream().findFirst().get()); - } - - /** - * Returns the outer docker network ip address and port of a container. This can be used to reach a - * container from the host machine - * - * @param container container - * @return a pair of host and port - */ - public static ImmutablePair getOuterContainerAddress(final Container container) { - return ImmutablePair.of( - container.getHost(), - container.getFirstMappedPort()); - } - } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshTunnel.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshTunnel.java index 42dbc0007df4ef..649f0576a589d8 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshTunnel.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshTunnel.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import javax.validation.constraints.NotNull; import org.apache.sshd.client.SshClient; import org.apache.sshd.client.keyverifier.AcceptAllServerKeyVerifier; import org.apache.sshd.client.session.ClientSession; @@ -33,7 +34,6 @@ import org.apache.sshd.common.util.security.SecurityUtils; import org.apache.sshd.core.CoreModuleProperties; import org.apache.sshd.server.forward.AcceptAllForwardingFilter; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperations.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperations.java similarity index 98% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperations.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperations.java index 5f4d47e6eb76b2..14d17485588a9d 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperations.java @@ -25,7 +25,7 @@ */ public interface SqlOperations { - Logger LOGGER = LoggerFactory.getLogger(JdbcBufferedConsumerFactory.class); + Logger LOGGER = LoggerFactory.getLogger(SqlOperations.class); /** * Create a schema with provided name if it does not already exist. diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/StagingFilenameGenerator.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/StagingFilenameGenerator.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/StagingFilenameGenerator.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/StagingFilenameGenerator.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopier.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopier.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopier.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopier.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopierFactory.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopierFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopierFactory.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopierFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java similarity index 97% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java index ae1b132de43faf..f91fada8b4653d 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java @@ -7,9 +7,9 @@ import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer; +import java.time.Instant; import java.util.List; import java.util.UUID; -import org.joda.time.DateTime; /** * Staging operations focuses on the SQL queries that are needed to success move data into a staging @@ -27,7 +27,7 @@ public interface StagingOperations extends SqlOperations { * raw table). Not all destinations use the table name in the staging path (e.g. Snowflake * simply uses a timestamp + UUID), but e.g. Redshift does rely on this to ensure uniqueness. */ - String getStagingPath(UUID connectionId, String namespace, String streamName, String outputTableName, DateTime writeDatetime); + String getStagingPath(UUID connectionId, String namespace, String streamName, String outputTableName, Instant writeDatetime); /** * Returns the staging environment's name diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index 23aa415dde00bd..9f28a3ffc22b0a 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.16.3 +version=0.17.0 diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/check/impl/CommonDatabaseCheckTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/check/impl/CommonDatabaseCheckTest.java index 37086e62028913..4b560ae4b87625 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/check/impl/CommonDatabaseCheckTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/check/impl/CommonDatabaseCheckTest.java @@ -39,7 +39,6 @@ void setup() { @AfterEach void cleanup() throws Exception { DataSourceFactory.close(dataSource); - dslContext.close(); container.stop(); } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java index fc5717ab04fa9c..cb5498b70487c5 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java @@ -44,7 +44,7 @@ import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumerTest.java index f06b67121d5b49..67bc7c7dc427c4 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumerTest.java @@ -51,7 +51,7 @@ import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/pre_migration_schema.txt b/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/pre_migration_schema.txt deleted file mode 100644 index 5094aa2fd4f642..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/pre_migration_schema.txt +++ /dev/null @@ -1,23 +0,0 @@ -create table "public"."airbyte_toy_migrations"( - "installed_rank" int4 not null, - "version" varchar(50) null, - "description" varchar(200) not null, - "type" varchar(20) not null, - "script" varchar(1000) not null, - "checksum" int4 null, - "installed_by" varchar(100) not null, - "installed_on" timestamp(29) not null default null, - "execution_time" int4 not null, - "success" bool not null, - constraint "airbyte_toy_migrations_pk" - primary key ("installed_rank") -); -create table "public"."toy_cars"( - "id" int8 generated by default as identity not null, - "value" varchar(50) null, - constraint "toy_cars_pkey" - primary key ("id") -); -create unique index "airbyte_toy_migrations_pk" on "public"."airbyte_toy_migrations"("installed_rank" asc); -create index "airbyte_toy_migrations_s_idx" on "public"."airbyte_toy_migrations"("success" asc); -create unique index "toy_cars_pkey" on "public"."toy_cars"("id" asc); diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema.sql b/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema.sql deleted file mode 100644 index d32999831ce504..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE - TABLE - IF NOT EXISTS TOY_CARS( - id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, - value VARCHAR(50) - ); diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema_dump.txt b/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema_dump.txt deleted file mode 100644 index efca08018f9fcd..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema_dump.txt +++ /dev/null @@ -1,24 +0,0 @@ -create table "public"."airbyte_toy_migrations"( - "installed_rank" int4 not null, - "version" varchar(50) null, - "description" varchar(200) not null, - "type" varchar(20) not null, - "script" varchar(1000) not null, - "checksum" int4 null, - "installed_by" varchar(100) not null, - "installed_on" timestamp(29) not null default null, - "execution_time" int4 not null, - "success" bool not null, - constraint "airbyte_toy_migrations_pk" - primary key ("installed_rank") -); -create table "public"."toy_cars"( - "id" int8 generated by default as identity not null, - "value" varchar(50) null, - "created_at" timestamp(29) not null default null, - constraint "toy_cars_pkey" - primary key ("id") -); -create unique index "airbyte_toy_migrations_pk" on "public"."airbyte_toy_migrations"("installed_rank" asc); -create index "airbyte_toy_migrations_s_idx" on "public"."airbyte_toy_migrations"("success" asc); -create unique index "toy_cars_pkey" on "public"."toy_cars"("id" asc); diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshBastionContainer.java b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/integrations/base/ssh/SshBastionContainer.java similarity index 76% rename from airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshBastionContainer.java rename to airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/integrations/base/ssh/SshBastionContainer.java index 07a1786f60dd14..344c7ed205a972 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshBastionContainer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/integrations/base/ssh/SshBastionContainer.java @@ -4,11 +4,6 @@ package io.airbyte.cdk.integrations.base.ssh; -import static io.airbyte.cdk.integrations.base.ssh.SshHelpers.getInnerContainerAddress; -import static io.airbyte.cdk.integrations.base.ssh.SshHelpers.getOuterContainerAddress; -import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod.SSH_KEY_AUTH; -import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH; - import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.cdk.integrations.util.HostPortResolver; @@ -16,11 +11,16 @@ import java.io.IOException; import java.util.List; import java.util.Objects; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.testcontainers.containers.Container; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.Network; import org.testcontainers.images.builder.ImageFromDockerfile; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod.SSH_KEY_AUTH; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH; + public class SshBastionContainer implements AutoCloseable { private static final String SSH_USER = "sshuser"; @@ -98,4 +98,30 @@ public GenericContainer getContainer() { return bastion; } + /** + * Returns the inner docker network ip address and port of a container. This can be used to reach a + * container from another container running on the same network + * + * @param container container + * @return a pair of host and port + */ + public static ImmutablePair getInnerContainerAddress(final Container container) { + return ImmutablePair.of( + container.getContainerInfo().getNetworkSettings().getNetworks().entrySet().stream().findFirst().get().getValue().getIpAddress(), + (Integer) container.getExposedPorts().stream().findFirst().get()); + } + + /** + * Returns the outer docker network ip address and port of a container. This can be used to reach a + * container from the host machine + * + * @param container container + * @return a pair of host and port + */ + public static ImmutablePair getOuterContainerAddress(final Container container) { + return ImmutablePair.of( + container.getHost(), + container.getFirstMappedPort()); + } + } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/HostPortResolver.java b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/integrations/util/HostPortResolver.java similarity index 84% rename from airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/HostPortResolver.java rename to airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/integrations/util/HostPortResolver.java index 7b8b4f4ccfc1fb..39baff64dc7b3c 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/HostPortResolver.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/integrations/util/HostPortResolver.java @@ -23,13 +23,6 @@ public static String resolveIpAddress(GenericContainer container) { return getIpAddress(container); } - public static String encodeValue(final String value) { - if (value != null) { - return URLEncoder.encode(value, StandardCharsets.UTF_8); - } - return null; - } - private static String getIpAddress(GenericContainer container) { return Objects.requireNonNull(container.getContainerInfo() .getNetworkSettings() diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/ContainerFactory.java b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/ContainerFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/ContainerFactory.java rename to airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/ContainerFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/DatabaseConnectionHelper.java b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/DatabaseConnectionHelper.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/DatabaseConnectionHelper.java rename to airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/DatabaseConnectionHelper.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/NonContainer.java b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/NonContainer.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/NonContainer.java rename to airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/NonContainer.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/TestDatabase.java b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/TestDatabase.java similarity index 99% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/TestDatabase.java rename to airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/TestDatabase.java index 7af23509389481..f8ef633ae98900 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/TestDatabase.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/testFixtures/java/io/airbyte/cdk/testutils/TestDatabase.java @@ -233,7 +233,6 @@ public B integrationTestConfigBuilder() { @Override public void close() { execSQL(this.cleanupSQL.stream()); - dslContext.close(); execInContainer(inContainerUndoBootstrapCmd()); } diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/build.gradle b/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/build.gradle new file mode 100644 index 00000000000000..349f3d1fd37c29 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/build.gradle @@ -0,0 +1,7 @@ +dependencies { + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + implementation project(':airbyte-cdk:java:airbyte-cdk:core') + + implementation 'com.google.cloud:google-cloud-bigquery:1.133.1' +} + diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryDatabase.java b/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryDatabase.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryDatabase.java rename to airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryDatabase.java diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryResultSet.java b/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryResultSet.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryResultSet.java rename to airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryResultSet.java diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQuerySourceOperations.java b/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/java/io/airbyte/cdk/db/bigquery/BigQuerySourceOperations.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQuerySourceOperations.java rename to airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/java/io/airbyte/cdk/db/bigquery/BigQuerySourceOperations.java diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/TempBigQueryJoolDatabaseImpl.java b/airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/java/io/airbyte/cdk/db/bigquery/TempBigQueryJoolDatabaseImpl.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/TempBigQueryJoolDatabaseImpl.java rename to airbyte-cdk/java/airbyte-cdk/datastore-bigquery/src/main/java/io/airbyte/cdk/db/bigquery/TempBigQueryJoolDatabaseImpl.java diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-mongo/build.gradle b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/build.gradle new file mode 100644 index 00000000000000..71b5ca5908c5fa --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/build.gradle @@ -0,0 +1,16 @@ +java { + // TODO: rewrite code to avoid javac wornings in the first place + compileJava { + options.compilerArgs += "-Xlint:-try,-unchecked" + } +} + +dependencies { + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + implementation project(':airbyte-cdk:java:airbyte-cdk:core') + + api 'org.mongodb:mongodb-driver-sync:4.10.2' + + testFixturesApi 'org.testcontainers:mongodb:1.19.0' +} + diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoDatabase.java b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoDatabase.java new file mode 100644 index 00000000000000..8b87aaf0eac014 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoDatabase.java @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.db.mongodb; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.mongodb.ConnectionString; +import com.mongodb.MongoConfigurationException; +import com.mongodb.ReadConcern; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.MongoIterable; +import io.airbyte.cdk.db.AbstractDatabase; +import io.airbyte.commons.exceptions.ConnectionErrorException; +import io.airbyte.commons.functional.CheckedFunction; +import io.airbyte.commons.util.MoreIterators; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; +import org.bson.BsonDocument; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MongoDatabase extends AbstractDatabase implements AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoDatabase.class); + private static final int BATCH_SIZE = 1000; + private static final String MONGO_RESERVED_COLLECTION_PREFIX = "system."; + + private final ConnectionString connectionString; + private final com.mongodb.client.MongoDatabase database; + private final MongoClient mongoClient; + + public MongoDatabase(final String connectionString, final String databaseName) { + try { + this.connectionString = new ConnectionString(connectionString); + mongoClient = MongoClients.create(this.connectionString); + database = mongoClient.getDatabase(databaseName); + } catch (final MongoConfigurationException e) { + LOGGER.error(e.getMessage(), e); + throw new ConnectionErrorException(String.valueOf(e.getCode()), e.getMessage(), e); + } catch (final Exception e) { + LOGGER.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + @Override + public void close() throws Exception { + mongoClient.close(); + } + + public com.mongodb.client.MongoDatabase getDatabase() { + return database; + } + + public MongoIterable getDatabaseNames() { + return mongoClient.listDatabaseNames(); + } + + public Set getCollectionNames() { + final MongoIterable collectionNames = database.listCollectionNames(); + if (collectionNames == null) { + return Collections.EMPTY_SET; + } + return MoreIterators.toSet(database.listCollectionNames().iterator()).stream() + .filter(c -> !c.startsWith(MONGO_RESERVED_COLLECTION_PREFIX)).collect(Collectors.toSet()); + } + + public MongoCollection getCollection(final String collectionName) { + return database.getCollection(collectionName) + .withReadConcern(ReadConcern.MAJORITY); + } + + public MongoCollection getOrCreateNewCollection(final String collectionName) { + final Set collectionNames = MoreIterators.toSet(database.listCollectionNames().iterator()); + if (!collectionNames.contains(collectionName)) { + database.createCollection(collectionName); + } + return database.getCollection(collectionName); + } + + @VisibleForTesting + public MongoCollection createCollection(final String name) { + database.createCollection(name); + return database.getCollection(name); + } + + @VisibleForTesting + public String getName() { + return database.getName(); + } + + public Stream read(final String collectionName, final List columnNames, final Optional filter) { + try { + final MongoCollection collection = database.getCollection(collectionName); + final MongoCursor cursor = collection + .find(filter.orElse(new BsonDocument())) + .batchSize(BATCH_SIZE) + .cursor(); + + return getStream(cursor, (document) -> MongoUtils.toJsonNode(document, columnNames)) + .onClose(() -> { + try { + cursor.close(); + } catch (final Exception e) { + throw new RuntimeException(e.getMessage(), e); + } + }); + + } catch (final Exception e) { + LOGGER.error("Exception attempting to read data from collection: {}, {}", collectionName, e.getMessage()); + throw new RuntimeException(e); + } + } + + private Stream getStream(final MongoCursor cursor, final CheckedFunction mapper) { + return StreamSupport.stream(new Spliterators.AbstractSpliterator<>(Long.MAX_VALUE, Spliterator.ORDERED) { + + @Override + public boolean tryAdvance(final Consumer action) { + try { + final Document document = cursor.tryNext(); + if (document == null) { + return false; + } + action.accept(mapper.apply(document)); + return true; + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + }, false); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoDatabaseException.java b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoDatabaseException.java new file mode 100644 index 00000000000000..c05844f816528d --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoDatabaseException.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.db.mongodb; + +public class MongoDatabaseException extends RuntimeException { + + public static final String MONGO_DATA_BASE_NOT_FOUND = "Data Base with given name - %s not found."; + + public MongoDatabaseException(final String databaseName) { + super(String.format(MONGO_DATA_BASE_NOT_FOUND, databaseName)); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoUtils.java b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoUtils.java new file mode 100644 index 00000000000000..ac607c1d29434b --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/MongoUtils.java @@ -0,0 +1,429 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.db.mongodb; + +import static java.util.Arrays.asList; +import static org.bson.BsonType.ARRAY; +import static org.bson.BsonType.DATE_TIME; +import static org.bson.BsonType.DECIMAL128; +import static org.bson.BsonType.DOCUMENT; +import static org.bson.BsonType.DOUBLE; +import static org.bson.BsonType.INT32; +import static org.bson.BsonType.INT64; +import static org.bson.BsonType.OBJECT_ID; +import static org.bson.BsonType.STRING; +import static org.bson.BsonType.TIMESTAMP; +import static org.bson.codecs.configuration.CodecRegistries.fromProviders; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.mongodb.DBRefCodecProvider; +import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoCollection; +import io.airbyte.cdk.db.DataTypeUtils; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.util.MoreIterators; +import io.airbyte.protocol.models.CommonField; +import io.airbyte.protocol.models.JsonSchemaType; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.bson.BsonBinary; +import org.bson.BsonDateTime; +import org.bson.BsonDocument; +import org.bson.BsonDocumentReader; +import org.bson.BsonDouble; +import org.bson.BsonInt32; +import org.bson.BsonInt64; +import org.bson.BsonReader; +import org.bson.BsonString; +import org.bson.BsonTimestamp; +import org.bson.BsonType; +import org.bson.Document; +import org.bson.codecs.BsonCodecProvider; +import org.bson.codecs.BsonValueCodecProvider; +import org.bson.codecs.DocumentCodecProvider; +import org.bson.codecs.IterableCodecProvider; +import org.bson.codecs.JsonObjectCodecProvider; +import org.bson.codecs.MapCodecProvider; +import org.bson.codecs.ValueCodecProvider; +import org.bson.codecs.configuration.CodecRegistry; +import org.bson.codecs.jsr310.Jsr310CodecProvider; +import org.bson.types.Decimal128; +import org.bson.types.ObjectId; +import org.bson.types.Symbol; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MongoUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoUtils.class); + + // Shared constants + public static final String MONGODB_SERVER_URL = "mongodb://%s%s:%s/%s?authSource=admin&ssl=%s"; + public static final String MONGODB_CLUSTER_URL = "mongodb+srv://%s%s/%s?retryWrites=true&w=majority&tls=true"; + public static final String MONGODB_REPLICA_URL = "mongodb://%s%s/%s?authSource=admin&directConnection=false&ssl=true"; + public static final String USER = "user"; + public static final String INSTANCE_TYPE = "instance_type"; + public static final String INSTANCE = "instance"; + public static final String CLUSTER_URL = "cluster_url"; + public static final String SERVER_ADDRESSES = "server_addresses"; + public static final String REPLICA_SET = "replica_set"; + + // MongodbDestination specific constants + public static final String AUTH_TYPE = "auth_type"; + public static final String AUTHORIZATION = "authorization"; + public static final String LOGIN_AND_PASSWORD = "login/password"; + public static final String AIRBYTE_DATA_HASH = "_airbyte_data_hash"; + + // MongodbSource specific constants + public static final String AUTH_SOURCE = "auth_source"; + public static final String PRIMARY_KEY = "_id"; + public static final Set ALLOWED_CURSOR_TYPES = Set.of(DOUBLE, STRING, DOCUMENT, OBJECT_ID, DATE_TIME, + INT32, TIMESTAMP, INT64, DECIMAL128); + + private static final String MISSING_TYPE = "missing"; + private static final String NULL_TYPE = "null"; + public static final String AIRBYTE_SUFFIX = "_aibyte_transform"; + private static final int DISCOVER_LIMIT = 10000; + private static final String ID = "_id"; + + public static JsonSchemaType getType(final BsonType dataType) { + return switch (dataType) { + case BOOLEAN -> JsonSchemaType.BOOLEAN; + case INT32, INT64, DOUBLE, DECIMAL128 -> JsonSchemaType.NUMBER; + case STRING, SYMBOL, BINARY, DATE_TIME, TIMESTAMP, OBJECT_ID, REGULAR_EXPRESSION, JAVASCRIPT -> JsonSchemaType.STRING; + case ARRAY -> JsonSchemaType.ARRAY; + case DOCUMENT, JAVASCRIPT_WITH_SCOPE -> JsonSchemaType.OBJECT; + default -> JsonSchemaType.STRING; + }; + } + + public static JsonNode toJsonNode(final Document document, final List columnNames) { + final ObjectNode objectNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); + formatDocument(document, objectNode, columnNames); + return objectNode; + } + + public static Object getBsonValue(final BsonType type, final String value) { + try { + return switch (type) { + case INT32 -> new BsonInt32(Integer.parseInt(value)); + case INT64 -> new BsonInt64(Long.parseLong(value)); + case DOUBLE -> new BsonDouble(Double.parseDouble(value)); + case DECIMAL128 -> Decimal128.parse(value); + case TIMESTAMP -> new BsonTimestamp((int) Instant.parse(value).getEpochSecond(), 0); + case DATE_TIME -> new BsonDateTime(Instant.parse(value).toEpochMilli()); + case OBJECT_ID -> new ObjectId(value); + case SYMBOL -> new Symbol(value); + case STRING -> new BsonString(value); + default -> value; + }; + } catch (final Exception e) { + LOGGER.error(String.format("Failed to get BsonValue for field type %s", type), e.getMessage()); + return value; + } + } + + public static CommonField nodeToCommonField(final TreeNode> node) { + final CommonField field = node.getData(); + if (node.hasChildren()) { + final List> subFields = node.getChildren().stream().map(MongoUtils::nodeToCommonField).toList(); + return new CommonField<>(field.getName(), field.getType(), subFields); + } else { + return new CommonField<>(field.getName(), field.getType()); + } + } + + private static void formatDocument(final Document document, final ObjectNode objectNode, final List columnNames) { + final BsonDocument bsonDocument = toBsonDocument(document); + try (final BsonReader reader = new BsonDocumentReader(bsonDocument)) { + readDocument(reader, objectNode, columnNames); + } catch (final Exception e) { + LOGGER.error("Exception while parsing BsonDocument: {}", e.getMessage()); + throw new RuntimeException(e); + } + } + + private static ObjectNode readDocument(final BsonReader reader, final ObjectNode jsonNodes, final List columnNames) { + reader.readStartDocument(); + while (reader.readBsonType() != BsonType.END_OF_DOCUMENT) { + final var fieldName = reader.readName(); + final var fieldType = reader.getCurrentBsonType(); + if (DOCUMENT.equals(fieldType)) { + // recursion in used to parse inner documents + jsonNodes.set(fieldName, readDocument(reader, (ObjectNode) Jsons.jsonNode(Collections.emptyMap()), columnNames)); + } else if (ARRAY.equals(fieldType)) { + jsonNodes.set(fieldName, readArray(reader, columnNames, fieldName)); + } else { + readField(reader, jsonNodes, columnNames, fieldName, fieldType); + } + transformToStringIfMarked(jsonNodes, columnNames, fieldName); + } + reader.readEndDocument(); + + return jsonNodes; + } + + /** + * Determines whether TLS/SSL should be enabled for a standalone instance of MongoDB. + */ + public static boolean tlsEnabledForStandaloneInstance(final JsonNode config, final JsonNode instanceConfig) { + return config.has(JdbcUtils.TLS_KEY) ? config.get(JdbcUtils.TLS_KEY).asBoolean() + : (instanceConfig.has(JdbcUtils.TLS_KEY) ? instanceConfig.get(JdbcUtils.TLS_KEY).asBoolean() : true); + } + + public static void transformToStringIfMarked(final ObjectNode jsonNodes, final List columnNames, final String fieldName) { + if (columnNames.contains(fieldName + AIRBYTE_SUFFIX)) { + final JsonNode data = jsonNodes.get(fieldName); + if (data != null) { + jsonNodes.remove(fieldName); + jsonNodes.put(fieldName + AIRBYTE_SUFFIX, data.isTextual() ? data.asText() : data.toString()); + } else { + LOGGER.debug("WARNING Field list out of sync, Document doesn't contain field: {}", fieldName); + } + } + } + + private static JsonNode readArray(final BsonReader reader, final List columnNames, final String fieldName) { + reader.readStartArray(); + final var elements = Lists.newArrayList(); + + while (reader.readBsonType() != BsonType.END_OF_DOCUMENT) { + final var arrayFieldType = reader.getCurrentBsonType(); + if (DOCUMENT.equals(arrayFieldType)) { + // recursion is used to read inner doc + elements.add(readDocument(reader, (ObjectNode) Jsons.jsonNode(Collections.emptyMap()), columnNames)); + } else if (ARRAY.equals(arrayFieldType)) { + // recursion is used to read inner array + elements.add(readArray(reader, columnNames, fieldName)); + } else { + final var element = readField(reader, (ObjectNode) Jsons.jsonNode(Collections.emptyMap()), columnNames, fieldName, arrayFieldType); + elements.add(element.get(fieldName)); + } + } + reader.readEndArray(); + return Jsons.jsonNode(MoreIterators.toList(elements.iterator())); + } + + private static ObjectNode readField(final BsonReader reader, + final ObjectNode o, + final List columnNames, + final String fieldName, + final BsonType fieldType) { + switch (fieldType) { + case BOOLEAN -> o.put(fieldName, reader.readBoolean()); + case INT32 -> o.put(fieldName, reader.readInt32()); + case INT64 -> o.put(fieldName, reader.readInt64()); + case DOUBLE -> o.put(fieldName, reader.readDouble()); + case DECIMAL128 -> o.put(fieldName, toDouble(reader.readDecimal128())); + case TIMESTAMP -> o.put(fieldName, DataTypeUtils.toISO8601StringWithMilliseconds(reader.readTimestamp().getValue())); + case DATE_TIME -> o.put(fieldName, DataTypeUtils.toISO8601StringWithMilliseconds(reader.readDateTime())); + case BINARY -> o.put(fieldName, toByteArray(reader.readBinaryData())); + case SYMBOL -> o.put(fieldName, reader.readSymbol()); + case STRING -> o.put(fieldName, reader.readString()); + case OBJECT_ID -> o.put(fieldName, toString(reader.readObjectId())); + case JAVASCRIPT -> o.put(fieldName, reader.readJavaScript()); + case JAVASCRIPT_WITH_SCOPE -> readJavaScriptWithScope(o, reader, fieldName, columnNames); + case REGULAR_EXPRESSION -> toString(reader.readRegularExpression()); + default -> reader.skipValue(); + } + return o; + } + + /** + * Gets 10.000 documents from collection, gathers all unique fields and its type. In case when one + * field has different types in 2 and more documents, the type is set to String. + * + * @param collection mongo collection + * @return map of unique fields and its type + */ + public static List>> getUniqueFields(final MongoCollection collection) { + final var allkeys = new HashSet<>(getFieldsName(collection)); + + return allkeys.stream().map(key -> { + final var types = getTypes(collection, key); + final var type = getUniqueType(types); + final var fieldNode = new TreeNode<>(new CommonField<>(transformName(types, key), type)); + if (type.equals(DOCUMENT)) { + setSubFields(collection, fieldNode, key); + } + return fieldNode; + }).toList(); + } + + /** + * If one field has different types in 2 and more documents, the name is transformed to + * 'name_aibyte_transform'. + * + * @param types list with field types + * @param name field name + * @return name + */ + private static String transformName(final List types, final String name) { + return types.size() != 1 ? name + AIRBYTE_SUFFIX : name; + } + + private static void setSubFields(final MongoCollection collection, + final TreeNode> parentNode, + final String pathToField) { + final var nestedKeys = getFieldsName(collection, pathToField); + nestedKeys.forEach(key -> { + final var types = getTypes(collection, pathToField + "." + key); + final var nestedType = getUniqueType(types); + final var childNode = parentNode.addChild(new CommonField<>(transformName(types, key), nestedType)); + if (nestedType.equals(DOCUMENT)) { + setSubFields(collection, childNode, pathToField + "." + key); + } + }); + } + + private static List getFieldsName(final MongoCollection collection) { + return getFieldsName(collection, "$ROOT"); + } + + private static List getFieldsName(final MongoCollection collection, final String fieldName) { + final AggregateIterable output = collection.aggregate(Arrays.asList( + new Document("$limit", DISCOVER_LIMIT), + new Document("$project", new Document("arrayofkeyvalue", new Document("$objectToArray", "$" + fieldName))), + new Document("$unwind", "$arrayofkeyvalue"), + new Document("$group", new Document(ID, null).append("allkeys", new Document("$addToSet", "$arrayofkeyvalue.k"))))); + if (output.cursor().hasNext()) { + return (List) output.cursor().next().get("allkeys"); + } else { + return Collections.emptyList(); + } + } + + private static List getTypes(final MongoCollection collection, final String name) { + final var fieldName = "$" + name; + final AggregateIterable output = collection.aggregate(Arrays.asList( + new Document("$limit", DISCOVER_LIMIT), + new Document("$project", new Document(ID, 0).append("fieldType", new Document("$type", fieldName))), + new Document("$group", new Document(ID, new Document("fieldType", "$fieldType")) + .append("count", new Document("$sum", 1))))); + final var listOfTypes = new ArrayList(); + final var cursor = output.cursor(); + while (cursor.hasNext()) { + final var type = ((Document) cursor.next().get(ID)).get("fieldType").toString(); + if (!MISSING_TYPE.equals(type) && !NULL_TYPE.equals(type)) { + listOfTypes.add(type); + } + } + if (listOfTypes.isEmpty()) { + listOfTypes.add(NULL_TYPE); + } + return listOfTypes; + } + + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") + private static BsonType getUniqueType(final List types) { + if (types.size() != 1) { + return BsonType.STRING; + } else { + final var type = types.get(0); + return getBsonTypeByTypeAlias(type); + } + } + + private static BsonType getBsonTypeByTypeAlias(final String typeAlias) { + return switch (typeAlias) { + case "object" -> BsonType.DOCUMENT; + case "double" -> BsonType.DOUBLE; + case "string" -> BsonType.STRING; + case "objectId" -> BsonType.OBJECT_ID; + case "array" -> BsonType.ARRAY; + case "binData" -> BsonType.BINARY; + case "bool" -> BsonType.BOOLEAN; + case "date" -> BsonType.DATE_TIME; + case "null" -> BsonType.NULL; + case "regex" -> BsonType.REGULAR_EXPRESSION; + case "dbPointer" -> BsonType.DB_POINTER; + case "javascript" -> BsonType.JAVASCRIPT; + case "symbol" -> BsonType.SYMBOL; + case "javascriptWithScope" -> BsonType.JAVASCRIPT_WITH_SCOPE; + case "int" -> BsonType.INT32; + case "timestamp" -> BsonType.TIMESTAMP; + case "long" -> BsonType.INT64; + case "decimal" -> BsonType.DECIMAL128; + default -> BsonType.STRING; + }; + } + + private static BsonDocument toBsonDocument(final Document document) { + try { + final CodecRegistry customCodecRegistry = + fromProviders(asList( + new ValueCodecProvider(), + new BsonValueCodecProvider(), + new DocumentCodecProvider(), + new IterableCodecProvider(), + new MapCodecProvider(), + new Jsr310CodecProvider(), + new JsonObjectCodecProvider(), + new BsonCodecProvider(), + new DBRefCodecProvider())); + + // Override the default codec registry + return document.toBsonDocument(BsonDocument.class, customCodecRegistry); + } catch (final Exception e) { + LOGGER.error("Exception while converting Document to BsonDocument: {}", e.getMessage()); + throw new RuntimeException(e); + } + } + + private static String toString(final Object value) { + return value == null ? null : value.toString(); + } + + private static Double toDouble(final Decimal128 value) { + return value == null ? null : value.doubleValue(); + } + + private static byte[] toByteArray(final BsonBinary value) { + return value == null ? null : value.getData(); + } + + private static void readJavaScriptWithScope(final ObjectNode o, final BsonReader reader, final String fieldName, final List columnNames) { + final var code = reader.readJavaScriptWithScope(); + final var scope = readDocument(reader, (ObjectNode) Jsons.jsonNode(Collections.emptyMap()), columnNames); + o.set(fieldName, Jsons.jsonNode(ImmutableMap.of("code", code, "scope", scope))); + } + + public enum MongoInstanceType { + + STANDALONE("standalone"), + REPLICA("replica"), + ATLAS("atlas"); + + private final String type; + + public String getType() { + return this.type; + } + + MongoInstanceType(final String type) { + this.type = type; + } + + public static MongoInstanceType fromValue(final String value) { + for (final MongoInstanceType instance : values()) { + if (value.equalsIgnoreCase(instance.type)) { + return instance; + } + } + throw new IllegalArgumentException("Unknown instance type value: " + value); + } + + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/TreeNode.java b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/TreeNode.java new file mode 100644 index 00000000000000..214d14c0f0995d --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/main/java/io/airbyte/cdk/db/mongodb/TreeNode.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.db.mongodb; + +import java.util.LinkedList; +import java.util.List; + +public class TreeNode { + + private T data; + private TreeNode parent; + private List> children; + + public TreeNode(final T data) { + this.data = data; + this.children = new LinkedList<>(); + } + + public TreeNode addChild(final T child) { + final TreeNode childNode = new TreeNode(child); + childNode.parent = this; + this.children.add(childNode); + return childNode; + } + + public boolean hasChildren() { + return children != null && !children.isEmpty(); + } + + public T getData() { + return data; + } + + public void setData(final T data) { + this.data = data; + } + + public TreeNode getParent() { + return parent; + } + + public void setParent(final TreeNode parent) { + this.parent = parent; + } + + public List> getChildren() { + return children; + } + + public void setChildren(final List> children) { + this.children = children; + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/test/java/io/airbyte/cdk/db/mongodb/MongoUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/test/java/io/airbyte/cdk/db/mongodb/MongoUtilsTest.java new file mode 100644 index 00000000000000..18f2828e0860cf --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/datastore-mongo/src/test/java/io/airbyte/cdk/db/mongodb/MongoUtilsTest.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.db.mongodb; + +import static io.airbyte.cdk.db.mongodb.MongoUtils.AIRBYTE_SUFFIX; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.json.Jsons; +import java.util.List; +import org.junit.jupiter.api.Test; + +class MongoUtilsTest { + + @Test + void testTransformToStringIfMarked() { + final List columnNames = List.of("_id", "createdAt", "connectedWallets", "connectedAccounts_aibyte_transform"); + final String fieldName = "connectedAccounts"; + final JsonNode node = Jsons.deserialize( + "{\"_id\":\"12345678as\",\"createdAt\":\"2022-11-11 12:13:14\",\"connectedWallets\":\"wallet1\"," + + "\"connectedAccounts\":" + + "{\"google\":{\"provider\":\"google\",\"refreshToken\":\"test-rfrsh-google-token-1\",\"accessToken\":\"test-access-google-token-1\",\"expiresAt\":\"2020-09-01T21:07:00Z\",\"createdAt\":\"2020-09-01T20:07:01Z\"}," + + + "\"figma\":{\"provider\":\"figma\",\"refreshToken\":\"test-rfrsh-figma-token-1\",\"accessToken\":\"test-access-figma-token-1\",\"expiresAt\":\"2020-12-13T22:08:03Z\",\"createdAt\":\"2020-09-14T22:08:03Z\",\"figmaInfo\":{\"teamID\":\"501087711831561793\"}}," + + + "\"slack\":{\"provider\":\"slack\",\"accessToken\":\"test-access-slack-token-1\",\"createdAt\":\"2020-09-01T20:15:07Z\",\"slackInfo\":{\"userID\":\"UM5AD2YCE\",\"teamID\":\"T2VGY5GH5\"}}}}"); + assertTrue(node.get(fieldName).isObject()); + + MongoUtils.transformToStringIfMarked((ObjectNode) node, columnNames, fieldName); + + assertNull(node.get(fieldName)); + assertNotNull(node.get(fieldName + AIRBYTE_SUFFIX)); + assertTrue(node.get(fieldName + AIRBYTE_SUFFIX).isTextual()); + + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-postgres/build.gradle b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/build.gradle new file mode 100644 index 00000000000000..3612172d1c3f73 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/build.gradle @@ -0,0 +1,9 @@ +dependencies { + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + implementation project(':airbyte-cdk:java:airbyte-cdk:core') + + api 'org.postgresql:postgresql:42.6.0' + + testFixturesApi 'org.testcontainers:postgresql:1.19.0' +} + diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/java/io/airbyte/cdk/db/PgLsn.java b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/java/io/airbyte/cdk/db/PgLsn.java new file mode 100644 index 00000000000000..eb7ab7013756c3 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/java/io/airbyte/cdk/db/PgLsn.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.db; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import java.nio.ByteBuffer; + +/** + * Doc on the structure of a Postgres LSN + * https://www.postgresql.org/docs/current/datatype-pg-lsn.html + */ +public class PgLsn implements Comparable { + + private final long lsn; + + public static PgLsn fromLong(final long lsn) { + return new PgLsn(lsn); + } + + public static PgLsn fromPgString(final String lsn) { + return new PgLsn(lsnToLong(lsn)); + } + + private PgLsn(final long lsn) { + this.lsn = lsn; + } + + public long asLong() { + return lsn; + } + + public String asPgString() { + return longToLsn(lsn); + } + + @Override + public int compareTo(final PgLsn o) { + return Long.compare(lsn, o.asLong()); + } + + /** + * The LSN returned by Postgres is a 64-bit integer represented as hex encoded 32-bit integers + * separated by a /. reference: https://github.com/davecramer/LogicalDecode + * + * @param lsn string representation as returned by postgres + * @return long representation of the lsn string. + */ + @VisibleForTesting + static long lsnToLong(final String lsn) { + final int slashIndex = lsn.lastIndexOf('/'); + Preconditions.checkArgument(slashIndex >= 0); + + final String logicalXLogStr = lsn.substring(0, slashIndex); + // parses as a long but then cast to int. this allows us to retain the full 32 bits of the integer + // as opposed to the reduced value of Integer.MAX_VALUE. + final int logicalXlog = (int) Long.parseLong(logicalXLogStr, 16); + final String segmentStr = lsn.substring(slashIndex + 1, lsn.length()); + final int segment = (int) Long.parseLong(segmentStr, 16); + + final ByteBuffer buf = ByteBuffer.allocate(8); + buf.putInt(logicalXlog); + buf.putInt(segment); + buf.position(0); + return buf.getLong(); + } + + @VisibleForTesting + static String longToLsn(final long long1) { + final int front = (int) (long1 >> 32); + final int back = (int) long1; + return (Integer.toHexString(front) + "/" + Integer.toHexString(back)).toUpperCase(); + } + + @Override + public String toString() { + return "PgLsn{" + + "lsn=" + lsn + + '}'; + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/java/io/airbyte/cdk/db/PostgresUtils.java b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/java/io/airbyte/cdk/db/PostgresUtils.java new file mode 100644 index 00000000000000..0b16eb5fed00af --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/java/io/airbyte/cdk/db/PostgresUtils.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.db; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Preconditions; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import java.sql.SQLException; +import java.util.List; + +public class PostgresUtils { + + public static PgLsn getLsn(final JdbcDatabase database) throws SQLException { + // pg version >= 10. For versions < 10 use query select * from pg_current_xlog_location() + final List jsonNodes = database + .bufferedResultSetQuery(conn -> conn.createStatement().executeQuery("select * from pg_current_wal_lsn()"), + resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); + + Preconditions.checkState(jsonNodes.size() == 1); + return PgLsn.fromPgString(jsonNodes.get(0).get("pg_current_wal_lsn").asText()); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/PostgresSslConnectionUtils.java b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/java/io/airbyte/cdk/integrations/util/PostgresSslConnectionUtils.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/PostgresSslConnectionUtils.java rename to airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/main/java/io/airbyte/cdk/integrations/util/PostgresSslConnectionUtils.java diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/PgLsnTest.java b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/test/java/io/airbyte/cdk/db/PgLsnTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/PgLsnTest.java rename to airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/test/java/io/airbyte/cdk/db/PgLsnTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/PostgresUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/test/java/io/airbyte/cdk/db/PostgresUtilsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/PostgresUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/test/java/io/airbyte/cdk/db/PostgresUtilsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/PostgreSQLContainerHelper.java b/airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/testFixtures/java/io/airbyte/cdk/testutils/PostgreSQLContainerHelper.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/PostgreSQLContainerHelper.java rename to airbyte-cdk/java/airbyte-cdk/datastore-postgres/src/testFixtures/java/io/airbyte/cdk/testutils/PostgreSQLContainerHelper.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle index ab74b0eaf9237c..9cb102a4847f6b 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle @@ -1,32 +1,44 @@ - java { + // TODO: rewrite code to avoid javac wornings in the first place compileJava { - options.compilerArgs += "-Xlint:-deprecation" + options.compilerArgs += "-Xlint:-deprecation,-removal" + } + compileTestFixturesJava { + options.compilerArgs += "-Xlint:-try" } } dependencies { - // Depends on core CDK classes (OK 👍) - implementation project(':airbyte-cdk:java:airbyte-cdk:core') - testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') - testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) + api 'org.apache.commons:commons-csv:1.10.0' + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + implementation project(':airbyte-cdk:java:airbyte-cdk:core') compileOnly project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') - testImplementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-api') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') + implementation 'io.aesy:datasize:1.0.0' - testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + testImplementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:dependencies')) + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:core') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:core')) + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) + + /* + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + testImplementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:dependencies')) + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:dependencies')) - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') + implementation project(':airbyte-cdk:java:airbyte-cdk:core') + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) + compileOnly project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') + testImplementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} @@ -90,4 +102,6 @@ dependencies { implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') {exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.slf4j', module: 'slf4j-reload4j'} testImplementation libs.junit.jupiter.system.stubs + + */ } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/WriteConfig.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/WriteConfig.java index fde85f704ec0e4..35b8380f70bb4f 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/WriteConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/WriteConfig.java @@ -5,8 +5,7 @@ package io.airbyte.cdk.integrations.destination.jdbc; import io.airbyte.protocol.models.v0.DestinationSyncMode; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; +import java.time.Instant; /** * Write configuration POJO (plain old java object) for all destinations extending @@ -20,7 +19,7 @@ public class WriteConfig { private final String tmpTableName; private final String outputTableName; private final DestinationSyncMode syncMode; - private final DateTime writeDatetime; + private final Instant writeDatetime; public WriteConfig(final String streamName, final String namespace, @@ -28,7 +27,7 @@ public WriteConfig(final String streamName, final String tmpTableName, final String outputTableName, final DestinationSyncMode syncMode) { - this(streamName, namespace, outputSchemaName, tmpTableName, outputTableName, syncMode, DateTime.now(DateTimeZone.UTC)); + this(streamName, namespace, outputSchemaName, tmpTableName, outputTableName, syncMode, Instant.now()); } public WriteConfig(final String streamName, @@ -37,7 +36,7 @@ public WriteConfig(final String streamName, final String tmpTableName, final String outputTableName, final DestinationSyncMode syncMode, - final DateTime writeDatetime) { + final Instant writeDatetime) { this.streamName = streamName; this.namespace = namespace; this.outputSchemaName = outputSchemaName; @@ -77,7 +76,7 @@ public DestinationSyncMode getSyncMode() { return syncMode; } - public DateTime getWriteDatetime() { + public Instant getWriteDatetime() { return writeDatetime; } diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialStagingConsumerFactory.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialStagingConsumerFactory.java index dc37391f8b0655..9f2b4c4c8f3450 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialStagingConsumerFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialStagingConsumerFactory.java @@ -24,12 +24,11 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.time.Instant; import java.util.List; import java.util.UUID; import java.util.function.Consumer; import java.util.function.Function; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,7 +48,7 @@ public class SerialStagingConsumerFactory { // in a previous attempt but failed to load to the warehouse for some reason (interrupted?) instead. // This would also allow other programs/scripts // to load (or reload backups?) in the connection's staging area to be loaded at the next sync. - private static final DateTime SYNC_DATETIME = DateTime.now(DateTimeZone.UTC); + private static final Instant SYNC_DATETIME = Instant.now(); public static final UUID RANDOM_CONNECTION_ID = UUID.randomUUID(); public AirbyteMessageConsumer create(final Consumer outputRecordCollector, diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java index d2aa70a1449c41..72231b4b40cfc8 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -82,7 +82,6 @@ import java.util.stream.Stream; import lombok.Builder; import lombok.Getter; -import org.joda.time.DateTime; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; @@ -1549,7 +1548,7 @@ public void testStressPerformance() throws Exception { while (true) { System.out.println( "currentStreamNumber=" + currentStreamNumber + ", currentRecordNumberForStream=" - + currentRecordNumberForStream + ", " + DateTime.now()); + + currentRecordNumberForStream + ", " + Instant.now()); try { Thread.sleep(10000); } catch (final InterruptedException e) { diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle b/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle index c08d588ad67bcf..5fbed0f365dc31 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle @@ -1,37 +1,16 @@ import org.jsonschema2pojo.SourceType -import org.jsoup.Jsoup - -buildscript { - dependencies { - // from standard-source-test: - classpath 'org.jsoup:jsoup:1.13.1' // for generateSourceTestDocs - } -} plugins { id "com.github.eirnym.js2p" version "1.0" - - id 'application' - id 'airbyte-integration-test-java' - id "java-library" - id "java-test-fixtures" // https://docs.gradle.org/current/userguide/java_testing.html#sec:java_test_fixtures } java { + // TODO: rewrite code to avoid javac wornings in the first place compileJava { options.compilerArgs += "-Xlint:-try,-rawtypes,-unchecked,-removal" } } -project.configurations { - // From `base-debezium`: - testFixturesImplementation.extendsFrom implementation - - // From source-jdbc - testFixturesImplementation.extendsFrom implementation - testFixturesRuntimeOnly.extendsFrom runtimeOnly -} - // Convert yaml to java: relationaldb.models jsonSchema2Pojo { sourceType = SourceType.YAMLSCHEMA @@ -48,20 +27,41 @@ jsonSchema2Pojo { } dependencies { + + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') implementation project(':airbyte-cdk:java:airbyte-cdk:core') - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') + implementation 'io.debezium:debezium-api:2.4.0.Final' + implementation 'io.debezium:debezium-embedded:2.4.0.Final' + implementation 'org.codehaus.plexus:plexus-utils:4.0.0' + + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:dependencies')) + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:core') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:core')) + + testFixturesImplementation 'net.sourceforge.argparse4j:argparse4j:0.9.0' + testFixturesImplementation 'io.swagger:swagger-annotations:1.6.13' + testFixturesImplementation 'org.hamcrest:hamcrest-all:1.3' + testFixturesImplementation 'org.junit.platform:junit-platform-launcher:1.10.1' - testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - testImplementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-api') - testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') + testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:core')) + testImplementation project(':airbyte-cdk:java:airbyte-cdk:datastore-postgres') + testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:datastore-postgres')) + + testImplementation 'uk.org.webcompere:system-stubs-jupiter:2.0.1' + + /* + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + testImplementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:dependencies')) + testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:core')) + testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:datastore-postgres')) + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:dependencies')) + implementation project(':airbyte-cdk:java:airbyte-cdk:core') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:core')) testFixturesImplementation "org.hamcrest:hamcrest-all:1.3" @@ -143,57 +143,6 @@ dependencies { testFixturesCompileOnly libs.testcontainers.postgresql testFixturesCompileOnly libs.testcontainers.cockroachdb testFixturesImplementation libs.testcontainers.cockroachdb -} - -def getFullPath(String className) { - def matchingFiles = project.fileTree("src/testFixtures/java") - .filter { file -> file.getName().equals("${className}.java".toString()) }.asCollection() - if (matchingFiles.size() == 0) { - throw new IllegalArgumentException("Ambiguous class name ${className}: no file found.") - } - if (matchingFiles.size() > 1) { - throw new IllegalArgumentException("Ambiguous class name ${className}: more than one matching file was found. Files found: ${matchingFiles}") - } - def absoluteFilePath = matchingFiles[0].toString() - def pathInPackage = project.relativePath(absoluteFilePath.toString()).replaceAll("src/testFixtures/java/", "").replaceAll("\\.java", "") - return pathInPackage -} - -def generateSourceTestDocs = tasks.register('generateSourceTestDocs', Javadoc) { - def javadocOutputDir = project.file("${project.buildDir}/docs/testFixturesJavadoc") - - options.addStringOption('Xdoclint:none', '-quiet') - classpath = sourceSets.testFixtures.compileClasspath - source = sourceSets.testFixtures.allJava - destinationDir = javadocOutputDir - - doLast { - def className = "SourceAcceptanceTest" - // this can be made into a list once we have multiple standard tests, and can also be used for destinations - def pathInPackage = getFullPath(className) - def stdSrcTest = project.file("${javadocOutputDir}/${pathInPackage}.html").readLines().join("\n") - def methodList = Jsoup.parse(stdSrcTest).body().select("section.methodDetails>ul>li>section") - def md = "" - for (methodInfo in methodList) { - def annotations = methodInfo.select(".memberSignature>.annotations").text() - if (!annotations.contains("@Test")) { - continue - } - def methodName = methodInfo.selectFirst("div>span.memberName").text() - def methodDocstring = methodInfo.selectFirst("div.block") - - md += "## ${methodName}\n\n" - md += "${methodDocstring != null ? methodDocstring.text().replaceAll(/([()])/, '\\\\$1') : 'No method description was provided'}\n\n" - } - def outputDoc = new File("${rootDir}/docs/connector-development/testing-connectors/standard-source-tests.md") - outputDoc.write "# Standard Source Test Suite\n\n" - outputDoc.append "Test methods start with `test`. Other methods are internal helpers in the java class implementing the test suite.\n\n" - outputDoc.append md - } - - outputs.upToDateWhen { false } -} -tasks.register('generate').configure { - dependsOn generateSourceTestDocs + */ } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/resources/spec.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/resources/spec.json deleted file mode 100644 index 4d668c68efb7bd..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/resources/spec.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/postgres", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "JDBC Source Spec", - "type": "object", - "required": ["username", "jdbc_url"], - "properties": { - "username": { - "title": "Username", - "description": "The username which is used to access the database.", - "type": "string" - }, - "password": { - "title": "Password", - "description": "The password associated with this username.", - "type": "string", - "airbyte_secret": true - }, - "jdbc_url": { - "title": "JDBC URL", - "description": "JDBC formatted URL. See the standard here.", - "type": "string" - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title": "JDBC URL Params", - "type": "string" - } - } - } -} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/dummy_config.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/dummy_config.json deleted file mode 100644 index 892b30269c606f..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/dummy_config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "username": "default", - "jdbc_url": "default" -} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/expected_spec.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/expected_spec.json deleted file mode 100644 index df19d77048be13..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/expected_spec.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/postgres", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "JDBC Source Spec", - "type": "object", - "required": ["username", "jdbc_url"], - "properties": { - "username": { - "title": "Username", - "description": "The username which is used to access the database.", - "type": "string" - }, - "password": { - "title": "Password", - "description": "The password associated with this username.", - "type": "string", - "airbyte_secret": true - }, - "jdbc_url": { - "title": "JDBC URL", - "description": "JDBC formatted URL. See the standard here.", - "type": "string" - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title": "JDBC URL Params", - "type": "string" - } - } - }, - "supported_destination_sync_modes": [] -} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-api/build.gradle b/airbyte-cdk/java/airbyte-cdk/dependencies/build.gradle similarity index 64% rename from airbyte-cdk/java/airbyte-cdk/airbyte-api/build.gradle rename to airbyte-cdk/java/airbyte-cdk/dependencies/build.gradle index ae117dba675003..8179d025ed6d1b 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-api/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/build.gradle @@ -1,13 +1,22 @@ import org.openapitools.generator.gradle.plugin.tasks.GenerateTask plugins { - id "org.openapi.generator" version "6.2.1" + id "com.github.eirnym.js2p" version "1.0" + id "de.undercouch.download" version "5.4.0" id "java-library" + id "org.openapi.generator" version "6.2.1" } java { + // TODO: rewrite code to avoid javac wornings in the first place compileJava { - options.compilerArgs += "-Xlint:-deprecation" + options.compilerArgs += "-Xlint:-varargs,-try,-deprecation,-unchecked" + } + compileTestJava { + options.compilerArgs += "-Xlint:-try" + } + compileTestFixturesJava { + options.compilerArgs += "-Xlint:-try" } } @@ -111,18 +120,6 @@ def generateApiDocs = tasks.register('generateApiDocs', GenerateTask) { generatePom : "false", interfaceOnly: "true" ]) - - doLast { - def target = file(rootProject.file("docs/reference/api/generated-api-html")) - delete target - mkdir target - copy { - from outputDir - include "**/*.html" - includeEmptyDirs = false - into target - } - } } def deleteExistingDocs = tasks.register('deleteOldApiDocs', Delete) { delete rootProject.file("docs/reference/api/generated-api-html") @@ -145,17 +142,61 @@ generate.configure { } dependencies { - implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310' + api platform('com.fasterxml.jackson:jackson-bom:2.15.2') + api 'com.fasterxml.jackson.core:jackson-annotations' + api 'com.fasterxml.jackson.core:jackson-databind' + api 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' + api 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' + api 'com.google.guava:guava:33.0.0-jre' + api 'commons-io:commons-io:2.15.1' + api ('io.airbyte.airbyte-protocol:protocol-models:0.5.0') { exclude group: 'com.google.api-client', module: 'google-api-client' } + api 'javax.annotation:javax.annotation-api:1.3.2' + api 'org.apache.commons:commons-compress:1.25.0' + api 'org.apache.commons:commons-lang3:3.14.0' + api 'org.apache.logging.log4j:log4j-api:2.21.1' + api 'org.apache.logging.log4j:log4j-core:2.21.1' + api 'org.apache.logging.log4j:log4j-slf4j-impl:2.21.1' + api 'org.apache.logging.log4j:log4j-slf4j2-impl:2.21.1' + api 'org.slf4j:log4j-over-slf4j:2.0.11' + api 'org.slf4j:slf4j-api:2.0.11' + + implementation 'com.jayway.jsonpath:json-path:2.7.0' + implementation 'com.networknt:json-schema-validator:1.0.72' + implementation 'commons-cli:commons-cli:1.4' + implementation 'io.swagger:swagger-annotations:1.6.2' + implementation 'javax.validation:validation-api:2.0.1.Final' + implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' + implementation 'me.andrz.jackson:jackson-json-reference-core:0.3.2' // needed so that we can follow $ref when parsing json + implementation 'org.openapitools:jackson-databind-nullable:0.2.1' + + testFixturesImplementation 'io.swagger:swagger-annotations:1.6.2' + testFixturesImplementation 'org.apache.ant:ant:1.10.11' + + testImplementation 'com.squareup.okhttp3:mockwebserver:4.9.1' +} - implementation group: 'io.swagger', name: 'swagger-annotations', version: '1.6.2' - implementation group: 'javax.annotation', name: 'javax.annotation-api', version: '1.3.2' - implementation group: 'javax.ws.rs', name: 'javax.ws.rs-api', version: '2.1.1' - implementation group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final' +jsonSchema2Pojo { + sourceType = org.jsonschema2pojo.SourceType.YAMLSCHEMA + source = files("${sourceSets.main.output.resourcesDir}/types") + targetDirectory = new File(project.buildDir, 'generated/src/gen/java/') - implementation group: 'org.openapitools', name: 'jackson-databind-nullable', version: '0.2.1' + targetPackage = 'io.airbyte.configoss' + useLongIntegers = true + + removeOldOutput = true + + generateBuilders = true + includeConstructors = false + includeSetters = true + serializable = true +} + +generate.configure { + dependsOn tasks.named('generateJsonSchema2Pojo') } + sourceSets { main { java { @@ -174,3 +215,6 @@ sourceSets { tasks.named('compileJava').configure { dependsOn generate } +tasks.named('compileTestJava').configure { + dependsOn generate +} diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/api/client/AirbyteApiClient.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/api/client/AirbyteApiClient.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/java/io/airbyte/api/client/PatchedLogsApi.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/api/client/PatchedLogsApi.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/java/io/airbyte/api/client/PatchedLogsApi.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/api/client/PatchedLogsApi.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/src/main/java/io/airbyte/commons/cli/Clis.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/cli/Clis.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/src/main/java/io/airbyte/commons/cli/Clis.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/cli/Clis.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/concurrency/CompletableFutures.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/concurrency/CompletableFutures.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/concurrency/CompletableFutures.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/concurrency/CompletableFutures.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/concurrency/VoidCallable.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/concurrency/VoidCallable.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/concurrency/VoidCallable.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/concurrency/VoidCallable.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/concurrency/WaitingUtils.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/concurrency/WaitingUtils.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/concurrency/WaitingUtils.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/concurrency/WaitingUtils.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/constants/AirbyteSecretConstants.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/constants/AirbyteSecretConstants.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/constants/AirbyteSecretConstants.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/constants/AirbyteSecretConstants.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/enums/Enums.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/enums/Enums.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/enums/Enums.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/enums/Enums.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/exceptions/ConfigErrorException.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/exceptions/ConfigErrorException.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/exceptions/ConfigErrorException.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/exceptions/ConfigErrorException.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/exceptions/ConnectionErrorException.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/exceptions/ConnectionErrorException.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/exceptions/ConnectionErrorException.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/exceptions/ConnectionErrorException.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/exceptions/SQLRuntimeException.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/exceptions/SQLRuntimeException.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/exceptions/SQLRuntimeException.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/exceptions/SQLRuntimeException.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagHelper.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/FeatureFlagHelper.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagHelper.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/FeatureFlagHelper.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/FeatureFlags.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/FeatureFlags.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagsWrapper.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/FeatureFlagsWrapper.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagsWrapper.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/features/FeatureFlagsWrapper.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedBiConsumer.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedBiConsumer.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedBiConsumer.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedBiConsumer.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedBiFunction.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedBiFunction.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedBiFunction.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedBiFunction.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedConsumer.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedConsumer.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedConsumer.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedConsumer.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedFunction.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedFunction.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedFunction.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedFunction.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedSupplier.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedSupplier.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedSupplier.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/functional/CheckedSupplier.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/io/IOs.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/io/IOs.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/io/IOs.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/io/IOs.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/io/LineGobbler.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/io/LineGobbler.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/jackson/MoreMappers.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/jackson/MoreMappers.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/jackson/MoreMappers.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/jackson/MoreMappers.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/json/JsonPaths.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/json/JsonPaths.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/json/JsonSchemas.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/json/JsonSchemas.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/json/Jsons.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/json/Jsons.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableConsumer.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/CloseableConsumer.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableConsumer.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/CloseableConsumer.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableQueue.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/CloseableQueue.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableQueue.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/CloseableQueue.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableShutdownHook.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/CloseableShutdownHook.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableShutdownHook.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/CloseableShutdownHook.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/Exceptions.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/Exceptions.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/Exceptions.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/Exceptions.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/MoreBooleans.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/MoreBooleans.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/MoreBooleans.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/lang/MoreBooleans.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/logging/LoggingHelper.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/logging/LoggingHelper.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/logging/MdcScope.java similarity index 97% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/logging/MdcScope.java index 21264d559ef3ee..5cb8d668f052d0 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/logging/MdcScope.java @@ -28,7 +28,7 @@ */ public class MdcScope implements AutoCloseable { - public final static MdcScope.Builder DEFAULT_BUILDER = new Builder(); + public final static Builder DEFAULT_BUILDER = new Builder(); private final Map originalContextMap; diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/map/MoreMaps.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/map/MoreMaps.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/map/MoreMaps.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/map/MoreMaps.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/DefaultProtocolSerializer.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/protocol/DefaultProtocolSerializer.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/DefaultProtocolSerializer.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/protocol/DefaultProtocolSerializer.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ProtocolSerializer.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/protocol/ProtocolSerializer.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ProtocolSerializer.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/protocol/ProtocolSerializer.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/resources/MoreResources.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/resources/MoreResources.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/resources/MoreResources.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/resources/MoreResources.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/AirbyteStreamStatusHolder.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/stream/AirbyteStreamStatusHolder.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/AirbyteStreamStatusHolder.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/stream/AirbyteStreamStatusHolder.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/AirbyteStreamUtils.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/stream/AirbyteStreamUtils.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/AirbyteStreamUtils.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/stream/AirbyteStreamUtils.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/MoreStreams.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/stream/MoreStreams.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/MoreStreams.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/stream/MoreStreams.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/StreamStatusUtils.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/stream/StreamStatusUtils.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/StreamStatusUtils.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/stream/StreamStatusUtils.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/string/Strings.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/string/Strings.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/string/Strings.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/string/Strings.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/text/Names.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/text/Names.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/text/Names.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/text/Names.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/text/Sqls.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/text/Sqls.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/text/Sqls.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/text/Sqls.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AirbyteStreamAware.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/AirbyteStreamAware.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AirbyteStreamAware.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/AirbyteStreamAware.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterator.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/AutoCloseableIterator.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterator.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/AutoCloseableIterator.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/CompositeIterator.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/CompositeIterator.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/CompositeIterator.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/CompositeIterator.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/DefaultAutoCloseableIterator.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/DefaultAutoCloseableIterator.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/DefaultAutoCloseableIterator.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/DefaultAutoCloseableIterator.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/LazyAutoCloseableIterator.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/LazyAutoCloseableIterator.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/LazyAutoCloseableIterator.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/LazyAutoCloseableIterator.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/MoreIterators.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/MoreIterators.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/MoreIterators.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/util/MoreIterators.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersion.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersion.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersion.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersion.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/AirbyteVersion.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/AirbyteVersion.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/Version.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/Version.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/Version.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/Version.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/VersionDeserializer.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/VersionDeserializer.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/VersionDeserializer.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/VersionDeserializer.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/VersionSerializer.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/VersionSerializer.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/VersionSerializer.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/version/VersionSerializer.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/yaml/Yamls.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/yaml/Yamls.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/yaml/Yamls.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/commons/yaml/Yamls.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/AirbyteConfig.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/AirbyteConfig.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/AirbyteConfig.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/AirbyteConfig.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/AirbyteConfigValidator.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/AirbyteConfigValidator.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/AirbyteConfigValidator.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/AirbyteConfigValidator.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/CatalogDefinitionsConfig.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/CatalogDefinitionsConfig.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/CatalogDefinitionsConfig.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/CatalogDefinitionsConfig.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/ConfigSchema.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/ConfigSchema.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/ConfigSchema.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/ConfigSchema.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/Configs.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/Configs.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/Configs.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/Configs.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/EnvConfigs.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/EnvConfigs.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/EnvConfigs.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/EnvConfigs.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/helpers/StateMessageHelper.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/helpers/StateMessageHelper.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/helpers/StateMessageHelper.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/helpers/StateMessageHelper.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitions.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitions.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitions.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitions.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/AbstractSchemaValidator.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/validation/json/AbstractSchemaValidator.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/AbstractSchemaValidator.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/validation/json/AbstractSchemaValidator.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/ConfigSchemaValidator.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/validation/json/ConfigSchemaValidator.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/ConfigSchemaValidator.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/validation/json/ConfigSchemaValidator.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonSchemaValidator.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/validation/json/JsonSchemaValidator.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonSchemaValidator.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/validation/json/JsonSchemaValidator.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonValidationException.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/validation/json/JsonValidationException.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonValidationException.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/java/io/airbyte/validation/json/JsonValidationException.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/openapi/config.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/openapi/config.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/openapi/config.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/openapi/config.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/resources/log4j2-test.xml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/log4j2-test.xml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/resources/log4j2-test.xml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/log4j2-test.xml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ActorDefinitionResourceRequirements.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ActorDefinitionResourceRequirements.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ActorDefinitionResourceRequirements.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ActorDefinitionResourceRequirements.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ActorType.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ActorType.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ActorType.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ActorType.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/AllowedHosts.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/AllowedHosts.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/AllowedHosts.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/AllowedHosts.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/CombinedConnectorCatalog.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/CombinedConnectorCatalog.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/CombinedConnectorCatalog.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/CombinedConnectorCatalog.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ConnectorJobOutput.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ConnectorJobOutput.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ConnectorJobOutput.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ConnectorJobOutput.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DataType.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DataType.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DataType.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DataType.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationConnection.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DestinationConnection.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationConnection.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DestinationConnection.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationOAuthParameter.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DestinationOAuthParameter.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationOAuthParameter.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DestinationOAuthParameter.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationSyncMode.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DestinationSyncMode.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationSyncMode.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DestinationSyncMode.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DockerImageSpec.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DockerImageSpec.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DockerImageSpec.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/DockerImageSpec.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/FailureReason.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/FailureReason.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/FailureReason.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/FailureReason.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobGetSpecConfig.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/JobGetSpecConfig.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobGetSpecConfig.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/JobGetSpecConfig.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobSyncConfig.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/JobSyncConfig.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobSyncConfig.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/JobSyncConfig.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobType.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/JobType.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobType.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/JobType.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NamespaceDefinitionType.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/NamespaceDefinitionType.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NamespaceDefinitionType.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/NamespaceDefinitionType.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NotificationType.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/NotificationType.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NotificationType.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/NotificationType.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorDbt.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/OperatorDbt.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorDbt.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/OperatorDbt.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorNormalization.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/OperatorNormalization.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorNormalization.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/OperatorNormalization.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorType.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/OperatorType.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorType.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/OperatorType.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorWebhook.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/OperatorWebhook.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorWebhook.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/OperatorWebhook.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ReplicationStatus.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ReplicationStatus.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ReplicationStatus.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ReplicationStatus.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ResourceRequirements.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ResourceRequirements.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ResourceRequirements.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/ResourceRequirements.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SourceConnection.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SourceConnection.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SourceConnection.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SourceConnection.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SourceOAuthParameter.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SourceOAuthParameter.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SourceOAuthParameter.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SourceOAuthParameter.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardCheckConnectionInput.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardCheckConnectionInput.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardCheckConnectionInput.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardCheckConnectionInput.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardCheckConnectionOutput.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardCheckConnectionOutput.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardCheckConnectionOutput.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardCheckConnectionOutput.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardDestinationDefinition.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardDestinationDefinition.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardDestinationDefinition.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardDestinationDefinition.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardDiscoverCatalogInput.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardDiscoverCatalogInput.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardDiscoverCatalogInput.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardDiscoverCatalogInput.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSourceDefinition.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardSourceDefinition.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSourceDefinition.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardSourceDefinition.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSyncInput.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardSyncInput.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSyncInput.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardSyncInput.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSyncOperation.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardSyncOperation.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSyncOperation.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StandardSyncOperation.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/State.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/State.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/State.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/State.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StateType.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StateType.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StateType.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StateType.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StateWrapper.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StateWrapper.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StateWrapper.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/StateWrapper.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SuggestedStreams.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SuggestedStreams.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SuggestedStreams.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SuggestedStreams.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SyncMode.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SyncMode.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SyncMode.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SyncMode.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SyncStats.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SyncStats.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SyncStats.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/SyncStats.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WebhookOperationConfigs.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/WebhookOperationConfigs.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WebhookOperationConfigs.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/WebhookOperationConfigs.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WebhookOperationSummary.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/WebhookOperationSummary.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WebhookOperationSummary.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/WebhookOperationSummary.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WorkerDestinationConfig.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/WorkerDestinationConfig.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WorkerDestinationConfig.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/WorkerDestinationConfig.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WorkerSourceConfig.yaml b/airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/WorkerSourceConfig.yaml similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WorkerSourceConfig.yaml rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/types/WorkerSourceConfig.yaml diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-api/src/test/java/io/airbyte/api/client/AirbyteApiClientTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/api/client/AirbyteApiClientTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-api/src/test/java/io/airbyte/api/client/AirbyteApiClientTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/api/client/AirbyteApiClientTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/src/test/java/io/airbyte/commons/cli/ClisTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/cli/ClisTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/src/test/java/io/airbyte/commons/cli/ClisTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/cli/ClisTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/concurrency/WaitingUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/concurrency/WaitingUtilsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/concurrency/WaitingUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/concurrency/WaitingUtilsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/enums/EnumsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/enums/EnumsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/enums/EnumsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/enums/EnumsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/features/FeatureFlagHelperTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/features/FeatureFlagHelperTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/features/FeatureFlagHelperTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/features/FeatureFlagHelperTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/io/IOsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/io/IOsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/io/IOsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/io/IOsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/io/LineGobblerTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/io/LineGobblerTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/io/LineGobblerTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/io/LineGobblerTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonPathsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/json/JsonPathsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonPathsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/json/JsonPathsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/json/JsonsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/json/JsonsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/CloseableShutdownHookTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/lang/CloseableShutdownHookTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/CloseableShutdownHookTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/lang/CloseableShutdownHookTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/ExceptionsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/lang/ExceptionsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/ExceptionsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/lang/ExceptionsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/MoreBooleansTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/lang/MoreBooleansTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/MoreBooleansTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/lang/MoreBooleansTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/map/MoreMapsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/map/MoreMapsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/map/MoreMapsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/map/MoreMapsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/stream/AirbyteStreamStatusHolderTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/stream/AirbyteStreamStatusHolderTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/stream/AirbyteStreamStatusHolderTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/stream/AirbyteStreamStatusHolderTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/stream/StreamStatusUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/stream/StreamStatusUtilsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/stream/StreamStatusUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/stream/StreamStatusUtilsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/string/StringsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/string/StringsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/string/StringsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/string/StringsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/text/NamesTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/text/NamesTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/text/NamesTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/text/NamesTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/text/SqlsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/text/SqlsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/text/SqlsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/text/SqlsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/CompositeIteratorTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/util/CompositeIteratorTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/CompositeIteratorTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/util/CompositeIteratorTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/DefaultAutoCloseableIteratorTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/util/DefaultAutoCloseableIteratorTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/DefaultAutoCloseableIteratorTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/util/DefaultAutoCloseableIteratorTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/LazyAutoCloseableIteratorTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/util/LazyAutoCloseableIteratorTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/LazyAutoCloseableIteratorTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/util/LazyAutoCloseableIteratorTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/version/VersionTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/version/VersionTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/version/VersionTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/version/VersionTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/yaml/YamlsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/yaml/YamlsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/yaml/YamlsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/commons/yaml/YamlsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/ConfigSchemaTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/configoss/ConfigSchemaTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/ConfigSchemaTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/configoss/ConfigSchemaTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/DataTypeEnumTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/configoss/DataTypeEnumTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/DataTypeEnumTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/configoss/DataTypeEnumTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitionsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitionsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitionsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitionsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/TestHarnessUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/TestHarnessUtilsTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/TestHarnessUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/TestHarnessUtilsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/CatalogClientConvertersTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/helper/CatalogClientConvertersTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/CatalogClientConvertersTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/helper/CatalogClientConvertersTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/ConnectorConfigUpdaterTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/helper/ConnectorConfigUpdaterTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/ConnectorConfigUpdaterTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/helper/ConnectorConfigUpdaterTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactoryTest.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactoryTest.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactoryTest.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactoryTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/composite_json_schema.json b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/composite_json_schema.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/composite_json_schema.json rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/composite_json_schema.json diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_all_types.json b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/json_with_all_types.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_all_types.json rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/json_with_all_types.json diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields.json b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/json_with_array_type_fields.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields.json rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/json_with_array_type_fields.json diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields_no_items.json b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/json_with_array_type_fields_no_items.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields_no_items.json rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/json_with_array_type_fields_no_items.json diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields_with_composites.json b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/json_with_array_type_fields_with_composites.json similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields_with_composites.json rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/json_schemas/json_with_array_type_fields_with_composites.json diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/resource_test b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/resource_test similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/resource_test rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/resource_test diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/resource_test_a b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/resource_test_a similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/resource_test_a rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/resource_test_a diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_a b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/subdir/resource_test_a similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_a rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/subdir/resource_test_a diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_sub b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/subdir/resource_test_sub similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_sub rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/subdir/resource_test_sub diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_sub_2 b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/subdir/resource_test_sub_2 similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_sub_2 rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/subdir/resource_test_sub_2 diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-with-version.jsonl b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/version-detection/logs-with-version.jsonl similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-with-version.jsonl rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/version-detection/logs-with-version.jsonl diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-without-spec-message.jsonl b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/version-detection/logs-without-spec-message.jsonl similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-without-spec-message.jsonl rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/version-detection/logs-without-spec-message.jsonl diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-without-version.jsonl b/airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/version-detection/logs-without-version.jsonl similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-without-version.jsonl rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/test/resources/version-detection/logs-without-version.jsonl diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/TestHarness.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/TestHarness.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/TestHarness.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/TestHarness.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/TestHarnessUtils.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/TestHarnessUtils.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/TestHarnessUtils.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/TestHarnessUtils.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/WorkerConstants.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/WorkerConstants.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/WorkerConstants.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/WorkerConstants.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/exception/TestHarnessException.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/exception/TestHarnessException.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/exception/TestHarnessException.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/exception/TestHarnessException.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/CheckConnectionTestHarness.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/CheckConnectionTestHarness.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/CheckConnectionTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/CheckConnectionTestHarness.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DbtTransformationRunner.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DbtTransformationRunner.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionTestHarness.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DefaultCheckConnectionTestHarness.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DefaultCheckConnectionTestHarness.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogTestHarness.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DefaultDiscoverCatalogTestHarness.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DefaultDiscoverCatalogTestHarness.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultGetSpecTestHarness.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DefaultGetSpecTestHarness.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultGetSpecTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DefaultGetSpecTestHarness.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DiscoverCatalogTestHarness.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DiscoverCatalogTestHarness.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DiscoverCatalogTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/DiscoverCatalogTestHarness.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/GetSpecTestHarness.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/GetSpecTestHarness.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/GetSpecTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/general/GetSpecTestHarness.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/CatalogClientConverters.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/helper/CatalogClientConverters.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/CatalogClientConverters.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/helper/CatalogClientConverters.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/ConnectorConfigUpdater.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/helper/ConnectorConfigUpdater.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/ConnectorConfigUpdater.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/helper/ConnectorConfigUpdater.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/EntrypointEnvChecker.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/helper/EntrypointEnvChecker.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/EntrypointEnvChecker.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/helper/EntrypointEnvChecker.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/FailureHelper.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/helper/FailureHelper.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/FailureHelper.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/helper/FailureHelper.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteDestination.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteDestination.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteDestination.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteDestination.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriter.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriter.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriter.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriter.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriterFactory.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriterFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriterFactory.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriterFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteProtocolPredicate.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteProtocolPredicate.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteProtocolPredicate.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteProtocolPredicate.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteSource.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteSource.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteSource.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteSource.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteStreamFactory.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteStreamFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteStreamFactory.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/AirbyteStreamFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriter.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriter.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriter.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriter.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriterFactory.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriterFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriterFactory.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriterFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteSource.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteSource.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java similarity index 99% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java index b13642781f22cd..2badef87b11ad8 100644 --- a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java @@ -15,9 +15,9 @@ import java.nio.charset.StandardCharsets; import java.text.CharacterIterator; import java.text.StringCharacterIterator; +import java.time.Instant; import java.util.Optional; import java.util.stream.Stream; -import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,7 +91,7 @@ public Stream create(final BufferedReader bufferedReader) { try { final String errorMessage = String.format( "Airbyte has received a message at %s UTC which is larger than %s (size: %s). The sync has been failed to prevent running out of memory.", - DateTime.now(), + Instant.now(), humanReadableByteCountSI(maxMemory), humanReadableByteCountSI(messageSize)); throw exceptionClass.get().getConstructor(String.class).newInstance(errorMessage); diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/HeartbeatMonitor.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/HeartbeatMonitor.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/HeartbeatMonitor.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/internal/HeartbeatMonitor.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/NormalizationAirbyteStreamFactory.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/normalization/NormalizationAirbyteStreamFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/NormalizationAirbyteStreamFactory.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/normalization/NormalizationAirbyteStreamFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/NormalizationRunner.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/normalization/NormalizationRunner.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/NormalizationRunner.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/normalization/NormalizationRunner.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/DockerProcessFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/DockerProcessFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/IntegrationLauncher.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/IntegrationLauncher.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/IntegrationLauncher.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/IntegrationLauncher.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/Metadata.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/Metadata.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/Metadata.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/Metadata.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/ProcessFactory.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/ProcessFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/ProcessFactory.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/process/ProcessFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/TestConfigHelpers.java b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/test_utils/TestConfigHelpers.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/TestConfigHelpers.java rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/java/io/airbyte/workers/test_utils/TestConfigHelpers.java diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/dbt_transformation_entrypoint.sh b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/dbt_transformation_entrypoint.sh similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/dbt_transformation_entrypoint.sh rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/dbt_transformation_entrypoint.sh diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/check.sh b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/entrypoints/sync/check.sh similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/check.sh rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/entrypoints/sync/check.sh diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/init.sh b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/entrypoints/sync/init.sh similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/init.sh rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/entrypoints/sync/init.sh diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/main.sh b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/entrypoints/sync/main.sh similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/main.sh rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/entrypoints/sync/main.sh diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/image_exists.sh b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/image_exists.sh similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/image_exists.sh rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/image_exists.sh diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/sshtunneling.sh b/airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/sshtunneling.sh similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/sshtunneling.sh rename to airbyte-cdk/java/airbyte-cdk/dependencies/src/testFixtures/resources/sshtunneling.sh diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle new file mode 100644 index 00000000000000..042566fb9b42b4 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle @@ -0,0 +1,7 @@ +dependencies { + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + implementation project(':airbyte-cdk:java:airbyte-cdk:core') + + implementation 'com.google.cloud:google-cloud-storage:2.17.2' +} + diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsConfig.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsConfig.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsConfig.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsConfig.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopier.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopier.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopier.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopier.java diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopierFactory.java b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopierFactory.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopierFactory.java rename to airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopierFactory.java diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/build.gradle b/airbyte-cdk/java/airbyte-cdk/init-oss/build.gradle deleted file mode 100644 index b41a176dc90510..00000000000000 --- a/airbyte-cdk/java/airbyte-cdk/init-oss/build.gradle +++ /dev/null @@ -1,5 +0,0 @@ -// TODO: remove this module stub entirely during CDK module consolidation - -plugins { - id "java-library" -} diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle index 60cca471e6a068..ef2369647f0c71 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle @@ -1,55 +1,42 @@ +java { + // TODO: rewrite code to avoid javac wornings in the first place + compileJava { + options.compilerArgs += "-Xlint:-try,-deprecation" + } + compileTestJava { + options.compilerArgs += "-Xlint:-try" + } + compileTestFixturesJava { + options.compilerArgs += "-Xlint:-deprecation" + } +} dependencies { + api 'com.amazonaws:aws-java-sdk-s3:1.12.647' + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') implementation project(':airbyte-cdk:java:airbyte-cdk:core') - implementation project(':airbyte-cdk:java:airbyte-cdk:db-destinations') implementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') - testImplementation project(':airbyte-cdk:java:airbyte-cdk:db-destinations') - testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:db-destinations') - testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:db-destinations')) - - - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-api') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - - testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - - testImplementation 'org.mockito:mockito-core:4.6.1' - - // Lombok - implementation 'org.projectlombok:lombok:1.18.20' - annotationProcessor 'org.projectlombok:lombok:1.18.20' - testFixturesImplementation 'org.projectlombok:lombok:1.18.20' - testFixturesAnnotationProcessor 'org.projectlombok:lombok:1.18.20' - - implementation ('org.apache.hadoop:hadoop-aws:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} - implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') {exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.slf4j', module: 'slf4j-reload4j'} - implementation group: 'com.hadoop.gplcompression', name: 'hadoop-lzo', version: '0.4.20' - implementation ('org.apache.hadoop:hadoop-common:3.3.3') { - exclude group: 'org.slf4j', module: 'slf4j-log4j12' - exclude group: 'org.slf4j', module: 'slf4j-reload4j' - } + implementation project(':airbyte-cdk:java:airbyte-cdk:db-destinations') - implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' - implementation 'org.apache.commons:commons-csv:1.4' - implementation ('org.apache.parquet:parquet-avro:1.12.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} - - implementation libs.bundles.junit - testImplementation libs.junit.jupiter.system.stubs - - -} - -java { - compileJava { - options.compilerArgs.remove("-Werror") - } - compileTestJava { - options.compilerArgs += "-Xlint:-try" - } + implementation 'com.github.alexmojaki:s3-stream-upload:2.2.4' + implementation 'org.apache.avro:avro:1.11.3' + implementation 'org.apache.commons:commons-csv:1.10.0' + implementation 'org.apache.commons:commons-text:1.11.0' + implementation 'org.apache.hadoop:hadoop-aws:3.3.6' + implementation 'org.apache.hadoop:hadoop-common:3.3.6' + implementation 'org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6' + implementation 'org.apache.parquet:parquet-avro:1.13.1' + + testImplementation 'org.mockito:mockito-inline:5.2.0' + + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:dependencies')) + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:core') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:core')) + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:db-destinations') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:db-destinations')) } diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/BaseGcsDestination.java similarity index 83% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/BaseGcsDestination.java index 3b209f6fe7c736..4e481be31089ab 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/BaseGcsDestination.java @@ -2,19 +2,17 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import static io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage.getErrorMessage; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.internal.SkipMd5CheckStrategy; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.integrations.BaseConnector; import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; import io.airbyte.cdk.integrations.base.Destination; -import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; import io.airbyte.cdk.integrations.destination.record_buffer.FileBuffer; import io.airbyte.cdk.integrations.destination.s3.S3BaseChecks; @@ -28,24 +26,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class GcsDestination extends BaseConnector implements Destination { +public abstract class BaseGcsDestination extends BaseConnector implements Destination { - private static final Logger LOGGER = LoggerFactory.getLogger(GcsDestination.class); + private static final Logger LOGGER = LoggerFactory.getLogger(BaseGcsDestination.class); public static final String EXPECTED_ROLES = "storage.multipartUploads.abort, storage.multipartUploads.create, " + "storage.objects.create, storage.objects.delete, storage.objects.get, storage.objects.list"; private final NamingConventionTransformer nameTransformer; - public GcsDestination() { + public BaseGcsDestination() { this.nameTransformer = new GcsNameTransformer(); } - - public static void main(final String[] args) throws Exception { - System.setProperty(SkipMd5CheckStrategy.DISABLE_GET_OBJECT_MD5_VALIDATION_PROPERTY, "true"); - System.setProperty(SkipMd5CheckStrategy.DISABLE_PUT_OBJECT_MD5_VALIDATION_PROPERTY, "true"); - new IntegrationRunner(new GcsDestination()).run(args); - } - @Override public AirbyteConnectionStatus check(final JsonNode config) { try { @@ -70,7 +61,7 @@ public AirbyteConnectionStatus check(final JsonNode config) { LOGGER.error("Exception attempting to access the Gcs bucket: {}. Please make sure you account has all of these roles: " + EXPECTED_ROLES, e); AirbyteTraceMessageUtility.emitConfigErrorTrace(e, e.getMessage()); return new AirbyteConnectionStatus() - .withStatus(AirbyteConnectionStatus.Status.FAILED) + .withStatus(Status.FAILED) .withMessage("Could not connect to the Gcs bucket with the provided configuration. \n" + e .getMessage()); } diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfig.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationConfig.java similarity index 90% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfig.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationConfig.java index 23a72598e1cd48..e3567d863735a3 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; @@ -15,9 +15,9 @@ import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; import io.airbyte.cdk.integrations.destination.s3.S3FormatConfigs; import io.airbyte.cdk.integrations.destination.s3.S3StorageOperations; -import io.airbyte.integrations.destination.gcs.credential.GcsCredentialConfig; -import io.airbyte.integrations.destination.gcs.credential.GcsCredentialConfigs; -import io.airbyte.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; +import io.airbyte.cdk.integrations.destination.gcs.credential.GcsCredentialConfig; +import io.airbyte.cdk.integrations.destination.gcs.credential.GcsCredentialConfigs; +import io.airbyte.cdk.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; /** * Currently we always reuse the S3 client for GCS. So the GCS config extends from the S3 config. diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsNameTransformer.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsNameTransformer.java similarity index 79% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsNameTransformer.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsNameTransformer.java index af2146bd742b8a..c2cce517070cec 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsNameTransformer.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsNameTransformer.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import io.airbyte.cdk.integrations.destination.s3.util.S3NameTransformer; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsStorageOperations.java similarity index 97% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsStorageOperations.java index 453b4d60e7df7e..7233ac4fdb69f0 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/GcsStorageOperations.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriter.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriter.java similarity index 91% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriter.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriter.java index e63b436a732f7a..0bd0e973f0e528 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriter.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.avro; +package io.airbyte.cdk.integrations.destination.gcs.avro; import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; @@ -16,9 +16,9 @@ import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig; import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.util.GcsUtils; -import io.airbyte.integrations.destination.gcs.writer.BaseGcsWriter; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.util.GcsUtils; +import io.airbyte.cdk.integrations.destination.gcs.writer.BaseGcsWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; @@ -26,7 +26,6 @@ import java.util.UUID; import org.apache.avro.Schema; import org.apache.avro.file.DataFileWriter; -import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericData.Record; import org.apache.avro.generic.GenericDatumWriter; import org.slf4j.Logger; @@ -40,7 +39,7 @@ public class GcsAvroWriter extends BaseGcsWriter implements DestinationFileWrite private final AvroRecordFactory avroRecordFactory; private final StreamTransferManager uploadManager; private final MultiPartOutputStream outputStream; - private final DataFileWriter dataFileWriter; + private final DataFileWriter dataFileWriter; private final String gcsFileLocation; private final String objectKey; @@ -98,7 +97,7 @@ public void write(final UUID id, final AirbyteRecordMessage recordMessage) throw @Override public void write(final JsonNode formattedData) throws IOException { - final GenericData.Record record = avroRecordFactory.getAvroRecord(formattedData); + final Record record = avroRecordFactory.getAvroRecord(formattedData); dataFileWriter.append(record); } diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialConfig.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialConfig.java similarity index 86% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialConfig.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialConfig.java index f8465486a9f8e7..0166337f33c68b 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.credential; +package io.airbyte.cdk.integrations.destination.gcs.credential; import io.airbyte.cdk.integrations.destination.s3.credential.BlobStorageCredentialConfig; import io.airbyte.cdk.integrations.destination.s3.credential.S3CredentialConfig; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialConfigs.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialConfigs.java similarity index 91% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialConfigs.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialConfigs.java index 8ce44901743a66..9241a6af94f224 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialConfigs.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialConfigs.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.credential; +package io.airbyte.cdk.integrations.destination.gcs.credential; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialType.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialType.java similarity index 63% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialType.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialType.java index c22b6069b61963..a44f77241e39aa 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialType.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsCredentialType.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.credential; +package io.airbyte.cdk.integrations.destination.gcs.credential; public enum GcsCredentialType { HMAC_KEY diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsHmacKeyCredentialConfig.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsHmacKeyCredentialConfig.java similarity index 95% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsHmacKeyCredentialConfig.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsHmacKeyCredentialConfig.java index e1521ad34bf015..18bc1da6df61db 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsHmacKeyCredentialConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/credential/GcsHmacKeyCredentialConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.credential; +package io.airbyte.cdk.integrations.destination.gcs.credential; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvWriter.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/csv/GcsCsvWriter.java similarity index 95% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvWriter.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/csv/GcsCsvWriter.java index 1e76838b8a856c..72d2c91af5c936 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvWriter.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/csv/GcsCsvWriter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.csv; +package io.airbyte.cdk.integrations.destination.gcs.csv; import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; @@ -15,8 +15,8 @@ import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvFormatConfig; import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.writer.BaseGcsWriter; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.writer.BaseGcsWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlWriter.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/jsonl/GcsJsonlWriter.java similarity index 94% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlWriter.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/jsonl/GcsJsonlWriter.java index 1d56bd357d1d38..3e46bd13cbe70c 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlWriter.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/jsonl/GcsJsonlWriter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.jsonl; +package io.airbyte.cdk.integrations.destination.gcs.jsonl; import alex.mojaki.s3upload.MultiPartOutputStream; import alex.mojaki.s3upload.StreamTransferManager; @@ -16,8 +16,8 @@ import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.writer.BaseGcsWriter; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.writer.BaseGcsWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/parquet/GcsParquetWriter.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/parquet/GcsParquetWriter.java similarity index 92% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/parquet/GcsParquetWriter.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/parquet/GcsParquetWriter.java index b77aec84d515d2..75564344cc66cc 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/parquet/GcsParquetWriter.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/parquet/GcsParquetWriter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.parquet; +package io.airbyte.cdk.integrations.destination.gcs.parquet; import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.JsonNode; @@ -11,9 +11,9 @@ import io.airbyte.cdk.integrations.destination.s3.avro.AvroRecordFactory; import io.airbyte.cdk.integrations.destination.s3.parquet.S3ParquetFormatConfig; import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; -import io.airbyte.integrations.destination.gcs.writer.BaseGcsWriter; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; +import io.airbyte.cdk.integrations.destination.gcs.writer.BaseGcsWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; @@ -22,7 +22,6 @@ import java.sql.Timestamp; import java.util.UUID; import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericData.Record; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -64,7 +63,7 @@ public GcsParquetWriter(final GcsDestinationConfig config, final S3ParquetFormatConfig formatConfig = (S3ParquetFormatConfig) config.getFormatConfig(); final Configuration hadoopConfig = getHadoopConfig(config); - this.parquetWriter = AvroParquetWriter.builder(HadoopOutputFile.fromPath(path, hadoopConfig)) + this.parquetWriter = AvroParquetWriter.builder(HadoopOutputFile.fromPath(path, hadoopConfig)) .withSchema(schema) .withCompressionCodec(formatConfig.getCompressionCodec()) .withRowGroupSize(formatConfig.getBlockSize()) diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsS3FileSystem.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/util/GcsS3FileSystem.java similarity index 90% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsS3FileSystem.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/util/GcsS3FileSystem.java index 1fa384557fc425..eb0978cb2c06ec 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsS3FileSystem.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/util/GcsS3FileSystem.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.util; +package io.airbyte.cdk.integrations.destination.gcs.util; import java.io.IOException; import org.apache.hadoop.fs.s3a.Retries; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsUtils.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/util/GcsUtils.java similarity index 97% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsUtils.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/util/GcsUtils.java index 6c305c177ec7de..1bb7606d096e84 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/util/GcsUtils.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.util; +package io.airbyte.cdk.integrations.destination.gcs.util; import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.cdk.integrations.destination.s3.avro.AvroConstants; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/writer/BaseGcsWriter.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/writer/BaseGcsWriter.java similarity index 97% rename from airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/writer/BaseGcsWriter.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/writer/BaseGcsWriter.java index 225f85851fd7cd..032ea65fadc9a0 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/writer/BaseGcsWriter.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/gcs/writer/BaseGcsWriter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.writer; +package io.airbyte.cdk.integrations.destination.gcs.writer; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; @@ -12,7 +12,7 @@ import io.airbyte.cdk.integrations.destination.s3.S3Format; import io.airbyte.cdk.integrations.destination.s3.util.S3OutputPathHelper; import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingConsumerFactory.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingConsumerFactory.java index bd6de3d24bee37..35fe4a65fef661 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingConsumerFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingConsumerFactory.java @@ -27,6 +27,7 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.time.Instant; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -35,8 +36,6 @@ import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +47,7 @@ public class StagingConsumerFactory extends SerialStagingConsumerFactory { private static final Logger LOGGER = LoggerFactory.getLogger(StagingConsumerFactory.class); - private static final DateTime SYNC_DATETIME = DateTime.now(DateTimeZone.UTC); + private static final Instant SYNC_DATETIME = Instant.now(); public SerializedAirbyteMessageConsumer createAsync(final Consumer outputRecordCollector, final JdbcDatabase database, diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfigTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationConfigTest.java similarity index 88% rename from airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationConfigTest.java index bbecfdd52e4379..d4252764384e71 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationConfigTest.java @@ -2,18 +2,18 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.gcs.credential.GcsCredentialConfig; +import io.airbyte.cdk.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.destination.gcs.credential.GcsCredentialConfig; -import io.airbyte.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; import java.io.IOException; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java similarity index 96% rename from airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java index ae91d60910be48..a535a4679c5672 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.avro; +package io.airbyte.cdk.integrations.destination.gcs.avro; import static com.amazonaws.services.s3.internal.Constants.MB; import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; @@ -11,12 +11,12 @@ import alex.mojaki.s3upload.StreamTransferManager; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.util.ConfigTestUtils; import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig; import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.util.ConfigTestUtils; import java.util.List; import org.apache.avro.file.CodecFactory; import org.apache.avro.file.DataFileConstants; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriterTest.java similarity index 88% rename from airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriterTest.java index bec533af2b549c..dca5629d5319f3 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/avro/GcsAvroWriterTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.avro; +package io.airbyte.cdk.integrations.destination.gcs.avro; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; @@ -12,10 +12,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.SyncMode; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java similarity index 93% rename from airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java index 5521f04a5dcfe6..d58946d0f8ba78 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.csv; +package io.airbyte.cdk.integrations.destination.gcs.csv; import static com.amazonaws.services.s3.internal.Constants.MB; import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; @@ -11,12 +11,12 @@ import alex.mojaki.s3upload.StreamTransferManager; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.util.ConfigTestUtils; import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; import io.airbyte.cdk.integrations.destination.s3.util.Flattening; import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.util.ConfigTestUtils; import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java similarity index 92% rename from airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java index 3db5d455daff08..b67c593b76a132 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.jsonl; +package io.airbyte.cdk.integrations.destination.gcs.jsonl; import static com.amazonaws.services.s3.internal.Constants.MB; import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; @@ -10,11 +10,11 @@ import alex.mojaki.s3upload.StreamTransferManager; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.util.ConfigTestUtils; +import io.airbyte.cdk.integrations.destination.gcs.util.ConfigTestUtils; import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/util/ConfigTestUtils.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/util/ConfigTestUtils.java similarity index 89% rename from airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/util/ConfigTestUtils.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/util/ConfigTestUtils.java index 078a78c5aec942..ef47148b802fd2 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/util/ConfigTestUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/gcs/util/ConfigTestUtils.java @@ -2,13 +2,13 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs.util; +package io.airbyte.cdk.integrations.destination.gcs.util; import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; public class ConfigTestUtils { diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/resources/test_config.json b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/resources/test_config.json similarity index 100% rename from airbyte-integrations/connectors/destination-gcs/src/test/resources/test_config.json rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/resources/test_config.json diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java similarity index 99% rename from airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java index f5178562d19b76..f34a2d21b77f11 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroTestDataComparator.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsAvroTestDataComparator.java similarity index 97% rename from airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroTestDataComparator.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsAvroTestDataComparator.java index 2dcb585cd770ed..97793a57758d14 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroTestDataComparator.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsAvroTestDataComparator.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseAvroDestinationAcceptanceTest.java similarity index 91% rename from airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroDestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseAvroDestinationAcceptanceTest.java index e19036bd5c4769..f855843de3ebf0 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseAvroDestinationAcceptanceTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; @@ -23,13 +23,12 @@ import org.apache.avro.Schema.Type; import org.apache.avro.file.DataFileReader; import org.apache.avro.file.SeekableByteArrayInput; -import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericData.Record; import org.apache.avro.generic.GenericDatumReader; -public class GcsAvroDestinationAcceptanceTest extends GcsAvroParquetDestinationAcceptanceTest { +public abstract class GcsBaseAvroDestinationAcceptanceTest extends GcsAvroParquetDestinationAcceptanceTest { - public GcsAvroDestinationAcceptanceTest() { + public GcsBaseAvroDestinationAcceptanceTest() { super(S3Format.AVRO); } @@ -64,7 +63,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, new GenericDatumReader<>())) { final ObjectReader jsonReader = MAPPER.reader(); while (dataFileReader.hasNext()) { - final GenericData.Record record = dataFileReader.next(); + final Record record = dataFileReader.next(); final byte[] jsonBytes = AvroConstants.JSON_CONVERTER.convertToJson(record); JsonNode jsonRecord = jsonReader.readTree(jsonBytes); jsonRecord = nameUpdater.getJsonWithOriginalFieldNames(jsonRecord); @@ -88,7 +87,7 @@ protected Map> retrieveDataTypesFromPersistedFiles(final Strin new SeekableByteArrayInput(object.getObjectContent().readAllBytes()), new GenericDatumReader<>())) { while (dataFileReader.hasNext()) { - final GenericData.Record record = dataFileReader.next(); + final Record record = dataFileReader.next(); final Map> actualDataTypes = getTypes(record); resultDataTypes.putAll(actualDataTypes); } diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvDestinationAcceptanceTest.java similarity index 93% rename from airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvDestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvDestinationAcceptanceTest.java index 7c7985498e0eb3..5b55c80c71912c 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvDestinationAcceptanceTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; @@ -28,9 +28,9 @@ import org.apache.commons.csv.CSVRecord; import org.apache.commons.csv.QuoteMode; -public class GcsCsvDestinationAcceptanceTest extends GcsDestinationAcceptanceTest { +public abstract class GcsBaseCsvDestinationAcceptanceTest extends GcsDestinationAcceptanceTest { - public GcsCsvDestinationAcceptanceTest() { + public GcsBaseCsvDestinationAcceptanceTest() { super(S3Format.CSV); } @@ -55,7 +55,7 @@ private static Map getFieldTypes(final JsonNode streamSchema) { final JsonNode fieldDefinitions = streamSchema.get("properties"); final Iterator> iterator = fieldDefinitions.fields(); while (iterator.hasNext()) { - final Map.Entry entry = iterator.next(); + final Entry entry = iterator.next(); JsonNode fieldValue = entry.getValue(); JsonNode typeValue = fieldValue.get("type") == null ? fieldValue.get("$ref") : fieldValue.get("type"); fieldTypes.put(entry.getKey(), typeValue.asText()); @@ -70,7 +70,7 @@ private static JsonNode getJsonNode(final Map input, final Map entry : input.entrySet()) { + for (final Entry entry : input.entrySet()) { final String key = entry.getKey(); if (key.equals(JavaBaseConstants.COLUMN_NAME_AB_ID) || key .equals(JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) { diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvGzipDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvGzipDestinationAcceptanceTest.java similarity index 86% rename from airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvGzipDestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvGzipDestinationAcceptanceTest.java index 075b9532a54fee..6bf3d81ea48d6b 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvGzipDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvGzipDestinationAcceptanceTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import com.amazonaws.services.s3.model.S3Object; import com.fasterxml.jackson.databind.JsonNode; @@ -16,7 +16,7 @@ import java.util.Map; import java.util.zip.GZIPInputStream; -public class GcsCsvGzipDestinationAcceptanceTest extends GcsCsvDestinationAcceptanceTest { +public abstract class GcsBaseCsvGzipDestinationAcceptanceTest extends GcsBaseCsvDestinationAcceptanceTest { @Override public ProtocolVersion getProtocolVersion() { diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlDestinationAcceptanceTest.java similarity index 91% rename from airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlDestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlDestinationAcceptanceTest.java index 4d9fb0e216360d..40e7bae0051f48 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlDestinationAcceptanceTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; @@ -19,9 +19,9 @@ import java.util.List; import java.util.Map; -public class GcsJsonlDestinationAcceptanceTest extends GcsDestinationAcceptanceTest { +public abstract class GcsBaseJsonlDestinationAcceptanceTest extends GcsDestinationAcceptanceTest { - public GcsJsonlDestinationAcceptanceTest() { + public GcsBaseJsonlDestinationAcceptanceTest() { super(S3Format.JSONL); } diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlGzipDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlGzipDestinationAcceptanceTest.java similarity index 85% rename from airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlGzipDestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlGzipDestinationAcceptanceTest.java index 152bf22d153563..2924aecc8d1e9a 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlGzipDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlGzipDestinationAcceptanceTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import com.amazonaws.services.s3.model.S3Object; import com.fasterxml.jackson.databind.JsonNode; @@ -15,7 +15,7 @@ import java.util.Map; import java.util.zip.GZIPInputStream; -public class GcsJsonlGzipDestinationAcceptanceTest extends GcsJsonlDestinationAcceptanceTest { +public abstract class GcsBaseJsonlGzipDestinationAcceptanceTest extends GcsBaseJsonlDestinationAcceptanceTest { @Override public ProtocolVersion getProtocolVersion() { diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsParquetDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseParquetDestinationAcceptanceTest.java similarity index 88% rename from airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsParquetDestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseParquetDestinationAcceptanceTest.java index 65b2e68e816143..3725e76e02eaf3 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsParquetDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsBaseParquetDestinationAcceptanceTest.java @@ -2,12 +2,13 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectReader; +import io.airbyte.cdk.integrations.destination.gcs.parquet.GcsParquetWriter; import io.airbyte.cdk.integrations.destination.s3.S3Format; import io.airbyte.cdk.integrations.destination.s3.avro.AvroConstants; import io.airbyte.cdk.integrations.destination.s3.avro.JsonFieldNameUpdater; @@ -16,7 +17,6 @@ import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.gcs.parquet.GcsParquetWriter; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -26,15 +26,14 @@ import java.util.Map; import java.util.Set; import org.apache.avro.Schema.Type; -import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericData.Record; import org.apache.hadoop.conf.Configuration; import org.apache.parquet.avro.AvroReadSupport; import org.apache.parquet.hadoop.ParquetReader; -public class GcsParquetDestinationAcceptanceTest extends GcsAvroParquetDestinationAcceptanceTest { +public abstract class GcsBaseParquetDestinationAcceptanceTest extends GcsAvroParquetDestinationAcceptanceTest { - public GcsParquetDestinationAcceptanceTest() { + public GcsBaseParquetDestinationAcceptanceTest() { super(S3Format.PARQUET); } @@ -72,11 +71,11 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, final var path = new org.apache.hadoop.fs.Path(uri); final Configuration hadoopConfig = GcsParquetWriter.getHadoopConfig(config); - try (final ParquetReader parquetReader = ParquetReader.builder(new AvroReadSupport<>(), path) + try (final ParquetReader parquetReader = ParquetReader.builder(new AvroReadSupport<>(), path) .withConf(hadoopConfig) .build()) { final ObjectReader jsonReader = MAPPER.reader(); - GenericData.Record record; + Record record; while ((record = parquetReader.read()) != null) { final byte[] jsonBytes = AvroConstants.JSON_CONVERTER.convertToJson(record); JsonNode jsonRecord = jsonReader.readTree(jsonBytes); @@ -101,10 +100,10 @@ protected Map> retrieveDataTypesFromPersistedFiles(final Strin final var path = new org.apache.hadoop.fs.Path(uri); final Configuration hadoopConfig = S3ParquetWriter.getHadoopConfig(config); - try (final ParquetReader parquetReader = ParquetReader.builder(new AvroReadSupport<>(), path) + try (final ParquetReader parquetReader = ParquetReader.builder(new AvroReadSupport<>(), path) .withConf(hadoopConfig) .build()) { - GenericData.Record record; + Record record; while ((record = parquetReader.read()) != null) { final Map> actualDataTypes = getTypes(record); resultDataTypes.putAll(actualDataTypes); diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationAcceptanceTest.java similarity index 96% rename from airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationAcceptanceTest.java index b78e0ef089d446..8a04bc424f299c 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationAcceptanceTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.gcs; +package io.airbyte.cdk.integrations.destination.gcs; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -46,7 +46,7 @@ *

  • Implement {@link #getFormatConfig} that returns a {@link S3FormatConfig}
    • *
  • Implement {@link #retrieveRecords} that returns the Json records for the test
    • * - * Under the hood, a {@link io.airbyte.integrations.destination.gcs.GcsDestinationConfig} is + * Under the hood, a {@link GcsDestinationConfig} is * constructed as follows: *
  • Retrieve the secrets from "secrets/config.json"
    • *
  • Get the GCS bucket path from the constructor
    • @@ -79,11 +79,6 @@ public ProtocolVersion getProtocolVersion() { return ProtocolVersion.V1; } - @Override - protected String getImageName() { - return "airbyte/destination-gcs:dev"; - } - @Override protected JsonNode getConfig() { return configJson; @@ -240,7 +235,7 @@ public void testCheckIncorrectHmacKeyAccessIdCredential() { ((ObjectNode) baseJson).put("credential", credential); ((ObjectNode) baseJson).set("format", getFormatConfig()); - final GcsDestination destination = new GcsDestination(); + final BaseGcsDestination destination = new BaseGcsDestination() {}; final AirbyteConnectionStatus status = destination.check(baseJson); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: SignatureDoesNotMatch;")); @@ -258,7 +253,7 @@ public void testCheckIncorrectHmacKeySecretCredential() { ((ObjectNode) baseJson).put("credential", credential); ((ObjectNode) baseJson).set("format", getFormatConfig()); - final GcsDestination destination = new GcsDestination(); + final BaseGcsDestination destination = new BaseGcsDestination() {}; final AirbyteConnectionStatus status = destination.check(baseJson); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: SignatureDoesNotMatch;")); @@ -270,10 +265,9 @@ public void testCheckIncorrectBucketCredential() { ((ObjectNode) baseJson).put("gcs_bucket_name", "fake_bucket"); ((ObjectNode) baseJson).set("format", getFormatConfig()); - final GcsDestination destination = new GcsDestination(); + final BaseGcsDestination destination = new BaseGcsDestination() {}; final AirbyteConnectionStatus status = destination.check(baseJson); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: NoSuchKey;")); } - } diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroParquetTestDataComparator.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3AvroParquetTestDataComparator.java similarity index 94% rename from airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroParquetTestDataComparator.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3AvroParquetTestDataComparator.java index 4fe9040168ebfa..3cbbc4bc433fab 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroParquetTestDataComparator.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3AvroParquetTestDataComparator.java @@ -12,7 +12,7 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -public class S3BaseAvroParquetTestDataComparator extends AdvancedTestDataComparator { +public class S3AvroParquetTestDataComparator extends AdvancedTestDataComparator { @Override protected boolean compareDateValues(String airbyteMessageValue, String destinationValue) { diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.java index a048fd69dbfb86..0ba7ff5af30b4d 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.java @@ -73,7 +73,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, @Override protected TestDataComparator getTestDataComparator() { - return new S3BaseAvroParquetTestDataComparator(); + return new S3AvroParquetTestDataComparator(); } @Override diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.java index 44abebe02905c4..d8a88f9591c68a 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.java @@ -78,7 +78,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, @Override protected TestDataComparator getTestDataComparator() { - return new S3BaseAvroParquetTestDataComparator(); + return new S3AvroParquetTestDataComparator(); } @Override diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.java index 11cb05dde76178..7fd53f2aa51393 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.java @@ -64,11 +64,6 @@ protected JsonNode getBaseConfigJson() { return Jsons.deserialize(IOs.readFile(Path.of(secretFilePath))); } - @Override - protected String getImageName() { - return "airbyte/destination-s3:dev"; - } - @Override protected JsonNode getConfig() { return configJson; diff --git a/airbyte-cdk/java/airbyte-cdk/settings.gradle b/airbyte-cdk/java/airbyte-cdk/settings.gradle index 976a33b328eaf7..ed322fc863640f 100644 --- a/airbyte-cdk/java/airbyte-cdk/settings.gradle +++ b/airbyte-cdk/java/airbyte-cdk/settings.gradle @@ -1,15 +1,13 @@ rootProject.name = 'airbyte' -include ':airbyte-cdk:java:airbyte-cdk:airbyte-commons' -include ':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation' -include ':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli' -include ':airbyte-cdk:java:airbyte-cdk:airbyte-commons-protocol' -include ':airbyte-cdk:java:airbyte-cdk:airbyte-api' -include ':airbyte-cdk:java:airbyte-cdk:config-models-oss' -include ':airbyte-cdk:java:airbyte-cdk:init-oss' -include ':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness' +include ':airbyte-cdk:java:airbyte-cdk:dependencies' include ':airbyte-cdk:java:airbyte-cdk:core' include ':airbyte-cdk:java:airbyte-cdk:db-sources' include ':airbyte-cdk:java:airbyte-cdk:db-destinations' include ':airbyte-cdk:java:airbyte-cdk:s3-destinations' include ':airbyte-cdk:java:airbyte-cdk:typing-deduping' +include ':airbyte-cdk:java:airbyte-cdk:azure-blob-storage-destinations' +include ':airbyte-cdk:java:airbyte-cdk:gcs-destinations' +include ':airbyte-cdk:java:airbyte-cdk:datastore-bigquery' +include ':airbyte-cdk:java:airbyte-cdk:datastore-mongo' +include ':airbyte-cdk:java:airbyte-cdk:datastore-postgres' diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle b/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle index fc3ba8f061e834..9ec539430fa0fe 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle @@ -1,26 +1,21 @@ -plugins { - id 'java-library' +java { + // TODO: rewrite code to avoid javac wornings in the first place + compileJava { + options.compilerArgs += "-Xlint:-rawtypes,-unchecked" + } + compileTestFixturesJava { + options.compilerArgs += "-Xlint:-varargs" + } } dependencies { + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') implementation project(':airbyte-cdk:java:airbyte-cdk:core') - testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness') - testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') - implementation group: 'commons-codec', name: 'commons-codec', version: '1.16.0' - implementation libs.jooq - testFixturesImplementation libs.airbyte.protocol + implementation 'commons-codec:commons-codec:1.16.0' - testFixturesImplementation(platform('org.junit:junit-bom:5.8.2')) - testFixturesImplementation 'org.junit.jupiter:junit-jupiter-api' - testFixturesImplementation 'org.junit.jupiter:junit-jupiter-params' - testFixturesImplementation 'org.mockito:mockito-core:4.6.1' -} - -java { - compileJava { - options.compilerArgs.remove("-Werror") - } + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:dependencies')) + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:core') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:core')) } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Sql.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Sql.java index f9744e34da96da..bc0c1940566b05 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Sql.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Sql.java @@ -7,7 +7,6 @@ import java.util.Collections; import java.util.List; import java.util.stream.Stream; -import org.elasticsearch.common.Strings; /** * Represents a list of SQL transactions, where each transaction consists of one or more SQL @@ -27,7 +26,7 @@ public record Sql(List> transactions) { if (transaction.isEmpty()) { throw new IllegalArgumentException("Transaction must not be empty"); } - if (transaction.stream().anyMatch(Strings::isNullOrEmpty)) { + if (transaction.stream().anyMatch(s -> s == null || s.isEmpty())) { throw new IllegalArgumentException("Transaction must not contain empty statements"); } }); @@ -104,7 +103,7 @@ public static Sql concat(final List sqls) { public static Sql create(final List> transactions) { return new Sql(transactions.stream() .map(transaction -> transaction.stream() - .filter(statement -> !Strings.isNullOrEmpty(statement)) + .filter(statement -> statement != null && !statement.isEmpty()) .map(statement -> { if (!statement.trim().endsWith(";")) { return statement + ";"; diff --git a/airbyte-integrations/bases/base-java/build.gradle b/airbyte-integrations/bases/base-java/build.gradle index a80a2274f156d7..0c2de175e2cc9b 100644 --- a/airbyte-integrations/bases/base-java/build.gradle +++ b/airbyte-integrations/bases/base-java/build.gradle @@ -1,27 +1,3 @@ plugins { - id 'java-library' id 'airbyte-docker-legacy' } - -dependencies { - implementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - - implementation 'commons-cli:commons-cli:1.4' - implementation 'net.i2p.crypto:eddsa:0.3.0' - implementation 'org.apache.sshd:sshd-mina:2.8.0' - // bouncycastle is pinned to version-match the transitive dependency from kubernetes client-java - // because a version conflict causes "parameter object not a ECParameterSpec" on ssh tunnel initiation - implementation 'org.bouncycastle:bcprov-jdk15on:1.66' - implementation 'org.bouncycastle:bcpkix-jdk15on:1.66' - implementation 'org.bouncycastle:bctls-jdk15on:1.66' - - implementation libs.jackson.annotations - implementation libs.testcontainers - implementation libs.testcontainers.jdbc - implementation libs.bundles.datadog - - testImplementation 'commons-lang:commons-lang:2.6' - implementation group: 'org.apache.logging.log4j', name: 'log4j-layout-template-json', version: '2.17.2' -} diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index 961ff66d00973a..2bec17fe33a9d8 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -6,24 +6,23 @@ plugins { } dependencies { - project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness') + testFixtures(project(':airbyte-cdk:java:airbyte-cdk:dependencies')) } // we need to access the sshtunneling script from airbyte-workers for ssh support def copySshScript = tasks.register('copySshScript', Copy) { - from "${project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness').buildDir}/resources/main" + from "${project(':airbyte-cdk:java:airbyte-cdk:dependencies').buildDir}/resources/testFixtures" into "${buildDir}" include "sshtunneling.sh" } copySshScript.configure { - dependsOn project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness').tasks.named('processResources') + dependsOn project(':airbyte-cdk:java:airbyte-cdk:dependencies').tasks.named('processTestFixturesResources') } // make sure the copy task above worked (if it fails, it fails silently annoyingly) def checkSshScriptCopy = tasks.register('checkSshScriptCopy') { doFirst { - assert file("${buildDir}/sshtunneling.sh").exists() : - "Copy of sshtunneling.sh failed, check that it is present in airbyte-workers." + assert file("${buildDir}/sshtunneling.sh").exists() : "Copy of sshtunneling.sh failed." } } checkSshScriptCopy.configure { diff --git a/airbyte-integrations/connectors-performance/destination-harness/build.gradle b/airbyte-integrations/connectors-performance/destination-harness/build.gradle index 25332d02ca27fa..e5a3c4ca264ae5 100644 --- a/airbyte-integrations/connectors-performance/destination-harness/build.gradle +++ b/airbyte-integrations/connectors-performance/destination-harness/build.gradle @@ -9,7 +9,7 @@ application { } dependencies { - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') implementation 'io.fabric8:kubernetes-client:5.12.2' implementation 'org.apache.commons:commons-lang3:3.11' @@ -18,7 +18,6 @@ dependencies { implementation 'junit:junit:4.13.1' implementation 'org.testng:testng:7.1.0' implementation 'org.junit.jupiter:junit-jupiter:5.8.1' - } //This is only needed because we're using some very old libraries from airbyte-commons that were not packaged correctly diff --git a/airbyte-integrations/connectors-performance/source-harness/build.gradle b/airbyte-integrations/connectors-performance/source-harness/build.gradle index 2cdfcc461d3ed4..abfeaa794b4185 100644 --- a/airbyte-integrations/connectors-performance/source-harness/build.gradle +++ b/airbyte-integrations/connectors-performance/source-harness/build.gradle @@ -9,7 +9,7 @@ application { } dependencies { - implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') implementation 'io.fabric8:kubernetes-client:5.12.2' implementation 'org.apache.commons:commons-lang3:3.11' diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index 6b023138d6aa2d..b9c8b36a6a9b7b 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -1,16 +1,13 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.15.1' - features = ['db-destinations', 's3-destinations', 'typing-deduping'] - useLocalCdk = false + cdkVersionRequired = '0.17.0' + features = ['db-destinations', 's3-destinations', 'typing-deduping', 'datastore-bigquery'] + useLocalCdk = true } -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.destination.bigquery.BigQueryDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0', @@ -26,37 +23,9 @@ application { ] } -airbyteJavaConnector.addCdkDependencies() - dependencies { - // TODO: Pull out common classes into CDK instead of depending on another destination - implementation project(':airbyte-integrations:connectors:destination-gcs') - + implementation 'com.codepoetics:protonpack:1.13' implementation 'com.google.cloud:google-cloud-bigquery:2.31.1' - implementation 'org.apache.commons:commons-lang3:3.11' - implementation 'org.apache.commons:commons-csv:1.4' + implementation 'com.google.cloud:google-cloud-storage:2.4.5' implementation 'org.apache.commons:commons-text:1.10.0' - - implementation group: 'com.google.cloud', name: 'google-cloud-storage', version: '2.4.5' - implementation group: 'com.codepoetics', name: 'protonpack', version: '1.13' - - implementation (libs.airbyte.protocol) { - exclude group: 'io.airbyte', module: 'airbyte-commons' - } - // implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} - - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-bigquery') - - // This dependency is required because GCSOperaitons is leaking S3Client interface to the BigQueryDestination. - implementation libs.aws.java.sdk.s3 -} - -configurations.all { - resolutionStrategy { - // at time of writing: deps.toml declares google-cloud-storage 2.17.2 - // which pulls in google-api-client:2.2.0 - // which conflicts with google-cloud-bigquery, which requires google-api-client:1.x - // google-cloud-storage is OK with downgrading to anything >=1.31.1. - force 'com.google.api-client:google-api-client:1.31.5' - } } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java index 463439d02ae733..6ac2a7eeb0e0e5 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java @@ -44,10 +44,10 @@ import io.airbyte.integrations.destination.bigquery.uploader.BigQueryUploaderFactory; import io.airbyte.integrations.destination.bigquery.uploader.UploaderType; import io.airbyte.integrations.destination.bigquery.uploader.config.UploaderConfig; -import io.airbyte.integrations.destination.gcs.GcsDestination; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.GcsNameTransformer; -import io.airbyte.integrations.destination.gcs.GcsStorageOperations; +import io.airbyte.cdk.integrations.destination.gcs.BaseGcsDestination; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.GcsNameTransformer; +import io.airbyte.cdk.integrations.destination.gcs.GcsStorageOperations; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -155,7 +155,7 @@ private AirbyteConnectionStatus checkGcsPermission(final JsonNode config) { .map(i -> REQUIRED_PERMISSIONS.get(Math.toIntExact(i.getIndex()))) .toList()); - final GcsDestination gcsDestination = new GcsDestination(); + final BaseGcsDestination gcsDestination = new BaseGcsDestination() {}; final JsonNode gcsJsonNodeConfig = BigQueryUtils.getGcsJsonNodeConfig(config); return gcsDestination.check(gcsJsonNodeConfig); } catch (final Exception e) { diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java index 4e79ae4fb23fdd..651033419a0b99 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java @@ -17,8 +17,8 @@ import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer; import io.airbyte.cdk.integrations.util.ConnectorExceptionUtil; import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.GcsStorageOperations; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.GcsStorageOperations; import java.util.HashSet; import java.util.List; import java.util.Set; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java index 3377acceb1dab8..22c51b3492b4e1 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java @@ -40,7 +40,7 @@ import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.time.Instant; import java.time.LocalDateTime; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java index a2be651ca6eb31..eca7948fae5ea6 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java @@ -24,6 +24,7 @@ import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import java.math.BigInteger; import java.util.Comparator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -166,4 +167,8 @@ public void execute(final Sql sql) throws InterruptedException { } } + @Override + public LinkedHashMap findExistingFinalTables(List streamIds) throws Exception { + return null; + } } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java index 68f56c0fe41a9b..abf1816a407ebe 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java @@ -35,7 +35,7 @@ import io.airbyte.commons.string.Strings; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import io.airbyte.integrations.destination.bigquery.typing_deduping.BigQuerySqlGenerator; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java index f17b12320adc3a..c17ab5b3681493 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java @@ -9,7 +9,7 @@ import com.amazonaws.services.s3.AmazonS3; import io.airbyte.cdk.integrations.base.DestinationConfig; import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.cdk.integrations.destination.gcs.GcsDestinationConfig; import java.nio.file.Path; import java.util.HashSet; import org.junit.jupiter.api.Disabled; diff --git a/airbyte-integrations/connectors/destination-gcs/build.gradle b/airbyte-integrations/connectors/destination-gcs/build.gradle index a5cc5eb33d2844..c7c7a046ebd38c 100644 --- a/airbyte-integrations/connectors/destination-gcs/build.gradle +++ b/airbyte-integrations/connectors/destination-gcs/build.gradle @@ -1,51 +1,17 @@ plugins { - id 'application' id 'airbyte-java-connector' } airbyteJavaConnector { - cdkVersionRequired = '0.2.0' - features = ['db-destinations'] - useLocalCdk = false + cdkVersionRequired = '0.17.0' + features = ['db-destinations', 's3-destinations', 'gcs-destinations'] + useLocalCdk = true } -//remove once upgrading the CDK version to 0.4.x or later -java { - compileJava { - options.compilerArgs.remove("-Werror") - } -} - -airbyteJavaConnector.addCdkDependencies() - application { mainClass = 'io.airbyte.integrations.destination.gcs.GcsDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - - implementation platform('com.amazonaws:aws-java-sdk-bom:1.12.14') - implementation 'com.google.cloud.bigdataoss:gcs-connector:hadoop3-2.2.1' - - // csv - implementation 'com.amazonaws:aws-java-sdk-s3:1.11.978' - implementation 'org.apache.commons:commons-csv:1.4' - implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' - - // parquet - implementation ('org.apache.hadoop:hadoop-common:3.3.3') { - exclude group: 'org.slf4j', module: 'slf4j-log4j12' - exclude group: 'org.slf4j', module: 'slf4j-reload4j' - } - implementation ('org.apache.hadoop:hadoop-aws:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} - implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') { - exclude group: 'org.slf4j', module: 'slf4j-log4j12' - exclude group: 'org.slf4j', module: 'slf4j-reload4j' - } - implementation ('org.apache.parquet:parquet-avro:1.12.0') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} - implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} - - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation 'org.xerial.snappy:snappy-java:1.1.8.4' } diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/GcsDestination.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/GcsDestination.java new file mode 100644 index 00000000000000..bcb0aa6e671fba --- /dev/null +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/GcsDestination.java @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination; + +import com.amazonaws.services.s3.internal.SkipMd5CheckStrategy; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.destination.gcs.BaseGcsDestination; + +public class GcsDestination extends BaseGcsDestination { + + public static void main(final String[] args) throws Exception { + System.setProperty(SkipMd5CheckStrategy.DISABLE_GET_OBJECT_MD5_VALIDATION_PROPERTY, "true"); + System.setProperty(SkipMd5CheckStrategy.DISABLE_PUT_OBJECT_MD5_VALIDATION_PROPERTY, "true"); + new IntegrationRunner(new GcsDestination()).run(args); + } +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsAvroDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsAvroDestinationAcceptanceTest.java new file mode 100644 index 00000000000000..6f3f422c3548ba --- /dev/null +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsAvroDestinationAcceptanceTest.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination; + +import io.airbyte.cdk.integrations.destination.gcs.GcsBaseAvroDestinationAcceptanceTest; + +public class GcsAvroDestinationAcceptanceTest extends GcsBaseAvroDestinationAcceptanceTest { + + @Override + protected String getImageName() { + return "airbyte/destination-gcs:dev"; + } + +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsCsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsCsvDestinationAcceptanceTest.java new file mode 100644 index 00000000000000..6555e975a57c5e --- /dev/null +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsCsvDestinationAcceptanceTest.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination; + +import io.airbyte.cdk.integrations.destination.gcs.GcsBaseCsvDestinationAcceptanceTest; + +public class GcsCsvDestinationAcceptanceTest extends GcsBaseCsvDestinationAcceptanceTest { + @Override + protected String getImageName() { + return "airbyte/destination-gcs:dev"; + } + +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsCsvGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsCsvGzipDestinationAcceptanceTest.java new file mode 100644 index 00000000000000..dc4b720d2f4309 --- /dev/null +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsCsvGzipDestinationAcceptanceTest.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination; + +import io.airbyte.cdk.integrations.destination.gcs.GcsBaseCsvGzipDestinationAcceptanceTest; + +public class GcsCsvGzipDestinationAcceptanceTest extends GcsBaseCsvGzipDestinationAcceptanceTest { + @Override + protected String getImageName() { + return "airbyte/destination-gcs:dev"; + } + +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsJsonlDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsJsonlDestinationAcceptanceTest.java new file mode 100644 index 00000000000000..78a3d8e642569e --- /dev/null +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsJsonlDestinationAcceptanceTest.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination; + +import io.airbyte.cdk.integrations.destination.gcs.GcsBaseJsonlDestinationAcceptanceTest; + +public class GcsJsonlDestinationAcceptanceTest extends GcsBaseJsonlDestinationAcceptanceTest { + @Override + protected String getImageName() { + return "airbyte/destination-gcs:dev"; + } + + +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsJsonlGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsJsonlGzipDestinationAcceptanceTest.java new file mode 100644 index 00000000000000..de604e183733bf --- /dev/null +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsJsonlGzipDestinationAcceptanceTest.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination; + +import io.airbyte.cdk.integrations.destination.gcs.GcsBaseJsonlGzipDestinationAcceptanceTest; + +public class GcsJsonlGzipDestinationAcceptanceTest extends GcsBaseJsonlGzipDestinationAcceptanceTest { + @Override + protected String getImageName() { + return "airbyte/destination-gcs:dev"; + } + +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsParquetDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsParquetDestinationAcceptanceTest.java new file mode 100644 index 00000000000000..558a9de11dfbc6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/GcsParquetDestinationAcceptanceTest.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination; + +import io.airbyte.cdk.integrations.destination.gcs.GcsBaseParquetDestinationAcceptanceTest; + +public class GcsParquetDestinationAcceptanceTest extends GcsBaseParquetDestinationAcceptanceTest { + @Override + protected String getImageName() { + return "airbyte/destination-gcs:dev"; + } + +} diff --git a/airbyte-integrations/connectors/source-redshift/build.gradle b/airbyte-integrations/connectors/source-redshift/build.gradle index 9d9fdf35dbecfe..31e30f1ec35505 100644 --- a/airbyte-integrations/connectors/source-redshift/build.gradle +++ b/airbyte-integrations/connectors/source-redshift/build.gradle @@ -35,7 +35,4 @@ dependencies { testImplementation 'org.apache.commons:commons-dbcp2:2.7.0' testImplementation 'org.hamcrest:hamcrest-all:1.3' integrationTestJavaImplementation libs.testcontainers.jdbc - testImplementation project.project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - - } diff --git a/build.gradle b/build.gradle index 80da7caf07ea67..705d7f98d67ca3 100644 --- a/build.gradle +++ b/build.gradle @@ -180,6 +180,7 @@ subprojects { subproj -> } apply plugin: 'java' + apply plugin: 'java-test-fixtures' apply plugin: 'jacoco' apply plugin: 'com.github.spotbugs' @@ -194,6 +195,10 @@ subprojects { subproj -> //deprecation and removal are removed from error since we should still test those constructs. options.compilerArgs += ["-Werror", "-Xlint:all,-serial,-processing,-rawtypes,-unchecked,-deprecation,-removal"] } + compileTestFixturesJava { + //rawtypes and unchecked are necessary for mockito + options.compilerArgs += ["-Werror", "-Xlint:all,-serial,-processing,-rawtypes,-unchecked"] + } } if (isConnectorProject(subproj)) { @@ -319,45 +324,27 @@ subprojects { subproj -> finalizedBy jacocoTestReportTask } - // TODO: These should be added to the CDK or to the individual projects that need them: - dependencies { - implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0")) - implementation(platform("org.glassfish.jersey:jersey-bom:2.31")) - - // version is handled by "com.fasterxml.jackson:jackson-bom:2.10.4", so we do not explicitly set it here. - implementation libs.bundles.jackson - implementation libs.guava - implementation libs.commons.io - implementation libs.bundles.apache - implementation libs.slf4j.api - - // SLF4J as a facade over Log4j2 required dependencies - implementation libs.bundles.log4j - implementation libs.appender.log4j2 - - // Bridges from other logging implementations to SLF4J - implementation libs.bundles.slf4j - // Lombok dependencies compileOnly libs.lombok annotationProcessor libs.lombok - testCompileOnly libs.lombok testAnnotationProcessor libs.lombok + testFixturesCompileOnly libs.lombok + testFixturesAnnotationProcessor libs.lombok + // JUnit dependencies testRuntimeOnly libs.junit.jupiter.engine testImplementation libs.bundles.junit testImplementation libs.assertj.core - testImplementation libs.junit.pioneer + testFixturesImplementation libs.bundles.junit + testFixturesImplementation libs.assertj.core + testFixturesImplementation libs.junit.pioneer // adds owasp plugin spotbugsPlugins libs.findsecbugs.plugin implementation libs.spotbugs.annotations - - // Airbyte dependencies. - implementation libs.airbyte.protocol } tasks.withType(SpotBugsTask).configureEach { diff --git a/buildSrc/src/main/groovy/airbyte-java-connector.gradle b/buildSrc/src/main/groovy/airbyte-java-connector.gradle index fd2064733568bb..b0d9a5a35480ad 100644 --- a/buildSrc/src/main/groovy/airbyte-java-connector.gradle +++ b/buildSrc/src/main/groovy/airbyte-java-connector.gradle @@ -22,7 +22,7 @@ class AirbyteJavaConnectorExtension { addCdkDependencies() } - static final List IMPLEMENTATION = [ + static final List PRE_V017_IMPLEMENTATION = [ 'airbyte-commons', 'airbyte-json-validation', 'airbyte-commons-cli', @@ -31,14 +31,14 @@ class AirbyteJavaConnectorExtension { 'init-oss', ] - static final List TEST_IMPLEMENTATION = [ + static final List PRE_V017_TEST_IMPLEMENTATION = [ 'airbyte-commons', 'airbyte-json-validation', 'airbyte-api', 'config-models-oss', ] - static final List INTEGRATION_TEST_IMPLEMENTATION = [ + static final List PRE_V017_INTEGRATION_TEST_IMPLEMENTATION = [ 'config-models-oss', 'init-oss', 'acceptance-test-harness', @@ -57,13 +57,52 @@ class AirbyteJavaConnectorExtension { project.dependencies { def dep = { useLocalCdk ? project.project(projectName(it)) : jarName(it) } def testFixturesDep = { useLocalCdk ? testFixtures(project.project(projectName(it))) : "${jarName(it)}:test-fixtures" } - - IMPLEMENTATION.each { - implementation dep(it) - testFixturesImplementation dep(it) + if (useLocalCdk || !cdkVersionRequired.matches("^0\\.[0-9]\\..*|^0\\.1[0-6]\\..*")) { + // v0.11+ module structure + implementation dep("dependencies") + testImplementation dep("dependencies") + testFixturesImplementation dep("dependencies") + integrationTestJavaImplementation dep("dependencies") + integrationTestJavaImplementation testFixturesDep("dependencies") + } else { + // pre-v0.11 module structure + implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0")) + implementation(platform("org.glassfish.jersey:jersey-bom:2.31")) + + implementation "com.fasterxml.jackson.core:jackson-annotations:2.15.2" + implementation "com.fasterxml.jackson.core:jackson-databind:2.15.2" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.2" + implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.2" + + implementation "com.google.guava:guava:31.1-jre" + implementation "commons-io:commons-io:2.7" + implementation "org.apache.commons:commons-compress:1.20" + implementation "org.apache.commons:commons-lang3:3.11" + implementation "org.slf4j:slf4j-api:2.0.9" + + // SLF4J as a facade over Log4j2 required dependencies + implementation "org.apache.logging.log4j:log4j-api:2.21.1" + implementation "org.apache.logging.log4j:log4j-core:2.21.1" + implementation "org.apache.logging.log4j:log4j-slf4j2-impl:2.21.1" + implementation "org.apache.logging.log4j:log4j-web:2.21.1" + implementation "com.therealvan:appender-log4j2:3.6.0" + + // Bridges from other logging implementations to SLF4J + implementation "org.slf4j:jcl-over-slf4j:2.0.9" + implementation "org.slf4j:jul-to-slf4j:2.0.9" + implementation "org.slf4j:log4j-over-slf4j:2.0.9" + + // Airbyte dependencies. + implementation "io.airbyte.airbyte-protocol:protocol-models:0.3.6" + + // CDK dependencies. + PRE_V017_IMPLEMENTATION.each { + implementation jarName(it) + testFixturesImplementation jarName(it) + } + PRE_V017_TEST_IMPLEMENTATION.each {testImplementation jarName(it) } + PRE_V017_INTEGRATION_TEST_IMPLEMENTATION.each {integrationTestJavaImplementation jarName(it) } } - TEST_IMPLEMENTATION.each {testImplementation dep(it) } - INTEGRATION_TEST_IMPLEMENTATION.each {integrationTestJavaImplementation dep(it) } (["core"] + features).each { implementation dep(it) testFixturesImplementation dep(it) @@ -85,6 +124,7 @@ class AirbyteJavaConnectorPlugin implements Plugin { @Override void apply(Project project) { + project.plugins.apply('application') project.plugins.apply('java-test-fixtures') project.plugins.apply(AirbyteIntegrationTestJavaPlugin) project.plugins.apply(AirbytePerformanceTestJavaPlugin) diff --git a/settings.gradle b/settings.gradle index 91cb8e6871f129..4754bee55575a6 100644 --- a/settings.gradle +++ b/settings.gradle @@ -163,19 +163,17 @@ include ':tools:code-generator' include ':airbyte-cdk:python' include ':airbyte-cdk:java:airbyte-cdk' -include ':airbyte-cdk:java:airbyte-cdk:airbyte-commons' -include ':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation' -include ':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli' -include ':airbyte-cdk:java:airbyte-cdk:airbyte-commons-protocol' -include ':airbyte-cdk:java:airbyte-cdk:airbyte-api' -include ':airbyte-cdk:java:airbyte-cdk:config-models-oss' -include ':airbyte-cdk:java:airbyte-cdk:init-oss' -include ':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness' +include ':airbyte-cdk:java:airbyte-cdk:dependencies' include ':airbyte-cdk:java:airbyte-cdk:core' include ':airbyte-cdk:java:airbyte-cdk:db-destinations' include ':airbyte-cdk:java:airbyte-cdk:db-sources' include ':airbyte-cdk:java:airbyte-cdk:s3-destinations' include ':airbyte-cdk:java:airbyte-cdk:typing-deduping' +include ':airbyte-cdk:java:airbyte-cdk:azure-blob-storage-destinations' +include ':airbyte-cdk:java:airbyte-cdk:gcs-destinations' +include ':airbyte-cdk:java:airbyte-cdk:datastore-bigquery' +include ':airbyte-cdk:java:airbyte-cdk:datastore-mongo' +include ':airbyte-cdk:java:airbyte-cdk:datastore-postgres' include ':airbyte-integrations:bases:base' include ':airbyte-integrations:bases:base-java'