From 47c584c4911a0a7fcbed9daef96d4bbef3c4d359 Mon Sep 17 00:00:00 2001 From: Stephane Geneix Date: Thu, 28 Mar 2024 18:08:05 -0700 Subject: [PATCH] enabling spotbugs for gcs-destinations submodule --- .../destination/DestinationAcceptanceTest.kt | 6 +++--- .../airbyte-cdk/gcs-destinations/build.gradle | 2 -- ...GcsAvroParquetDestinationAcceptanceTest.kt | 11 +++++----- .../GcsBaseAvroDestinationAcceptanceTest.kt | 16 +++++++-------- .../GcsBaseCsvDestinationAcceptanceTest.kt | 4 ++-- .../GcsBaseJsonlDestinationAcceptanceTest.kt | 4 ++-- ...GcsBaseParquetDestinationAcceptanceTest.kt | 18 ++++++++--------- .../gcs/GcsDestinationAcceptanceTest.kt | 20 ++++++++++--------- .../s3/S3BaseAvroDestinationAcceptanceTest.kt | 4 ++-- .../s3/S3BaseCsvDestinationAcceptanceTest.kt | 4 ++-- .../S3BaseJsonlDestinationAcceptanceTest.kt | 4 ++-- .../S3BaseParquetDestinationAcceptanceTest.kt | 4 ++-- 12 files changed, 49 insertions(+), 48 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt index ca5a0525241cd..a8df1e140cb36 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.kt @@ -178,8 +178,8 @@ abstract class DestinationAcceptanceTest { @Throws(Exception::class) protected abstract fun retrieveRecords( testEnv: TestDestinationEnv?, - streamName: String?, - namespace: String?, + streamName: String, + namespace: String, streamSchema: JsonNode ): List @@ -1454,7 +1454,7 @@ abstract class DestinationAcceptanceTest { false ) val destinationOutput = - retrieveRecords(testEnv, stream.name, getDefaultSchema(config), stream.jsonSchema) + retrieveRecords(testEnv, stream.name, getDefaultSchema(config)!!, stream.jsonSchema) // Remove state message secondSyncMessagesWithNewFields.removeIf { airbyteMessage: io.airbyte.protocol.models.v0.AirbyteMessage -> diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle index 277fd6aeaace2..dbb42326adea9 100644 --- a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/build.gradle @@ -20,8 +20,6 @@ compileTestFixturesKotlin { } } -spotbugsTestFixtures.enabled = false - dependencies { implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-dependencies') implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-cdk-core') diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.kt index 40c4a5b4c48de..f1cd685917e76 100644 --- a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.kt @@ -90,13 +90,14 @@ abstract class GcsAvroParquetDestinationAcceptanceTest(s3Format: S3Format) : else fieldDefinition["type"] val airbyteTypeProperty = fieldDefinition["airbyte_type"] val airbyteTypePropertyText = airbyteTypeProperty?.asText() - return Arrays.stream(JsonSchemaType.entries.toTypedArray()) + return JsonSchemaType.entries + .toTypedArray() .filter { value: JsonSchemaType -> value.jsonSchemaType == typeProperty.asText() && compareAirbyteTypes(airbyteTypePropertyText, value) } - .map(JsonSchemaType::avroType) - .collect(Collectors.toSet()) + .map { it.avroType } + .toSet() } private fun compareAirbyteTypes( @@ -126,8 +127,8 @@ abstract class GcsAvroParquetDestinationAcceptanceTest(s3Format: S3Format) : @Throws(Exception::class) protected abstract fun retrieveDataTypesFromPersistedFiles( - streamName: String?, - namespace: String? + streamName: String, + namespace: String ): Map?> protected fun getTypes(record: GenericData.Record): Map> { diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseAvroDestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseAvroDestinationAcceptanceTest.kt index e10cbe9a46a5d..878d79abc9906 100644 --- a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseAvroDestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseAvroDestinationAcceptanceTest.kt @@ -35,17 +35,17 @@ abstract class GcsBaseAvroDestinationAcceptanceTest : @Throws(Exception::class) override fun retrieveRecords( testEnv: TestDestinationEnv?, - streamName: String?, - namespace: String?, + streamName: String, + namespace: String, streamSchema: JsonNode ): List { - val nameUpdater = getFieldNameUpdater(streamName!!, namespace, streamSchema) + val nameUpdater = getFieldNameUpdater(streamName, namespace, streamSchema) - val objectSummaries = getAllSyncedObjects(streamName, namespace) + val objectSummaries = getAllSyncedObjects(streamName, namespace!!) val jsonRecords: MutableList = LinkedList() - for (objectSummary in objectSummaries!!) { - val `object` = s3Client!!.getObject(objectSummary!!.bucketName, objectSummary.key) + for (objectSummary in objectSummaries) { + val `object` = s3Client.getObject(objectSummary.bucketName, objectSummary.key) DataFileReader( SeekableByteArrayInput(`object`.objectContent.readAllBytes()), GenericDatumReader() @@ -67,8 +67,8 @@ abstract class GcsBaseAvroDestinationAcceptanceTest : @Throws(Exception::class) override fun retrieveDataTypesFromPersistedFiles( - streamName: String?, - namespace: String? + streamName: String, + namespace: String ): Map?> { val objectSummaries = getAllSyncedObjects(streamName, namespace) val resultDataTypes: MutableMap?> = HashMap() diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvDestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvDestinationAcceptanceTest.kt index 8c86a2dc2e739..55f4767e4da90 100644 --- a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvDestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseCsvDestinationAcceptanceTest.kt @@ -40,8 +40,8 @@ abstract class GcsBaseCsvDestinationAcceptanceTest : GcsDestinationAcceptanceTes @Throws(IOException::class) override fun retrieveRecords( testEnv: TestDestinationEnv?, - streamName: String?, - namespace: String?, + streamName: String, + namespace: String, streamSchema: JsonNode ): List { val objectSummaries = getAllSyncedObjects(streamName, namespace) diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlDestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlDestinationAcceptanceTest.kt index b1d8d1d165e5e..4627425e4c9a8 100644 --- a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlDestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseJsonlDestinationAcceptanceTest.kt @@ -36,8 +36,8 @@ abstract class GcsBaseJsonlDestinationAcceptanceTest : @Throws(IOException::class) override fun retrieveRecords( testEnv: TestDestinationEnv?, - streamName: String?, - namespace: String?, + streamName: String, + namespace: String, streamSchema: JsonNode ): List { val objectSummaries = getAllSyncedObjects(streamName, namespace) diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseParquetDestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseParquetDestinationAcceptanceTest.kt index 36aa8f28a489a..004fc3ceed6c2 100644 --- a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseParquetDestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsBaseParquetDestinationAcceptanceTest.kt @@ -37,17 +37,17 @@ abstract class GcsBaseParquetDestinationAcceptanceTest : @Throws(IOException::class, URISyntaxException::class) override fun retrieveRecords( testEnv: TestDestinationEnv?, - streamName: String?, - namespace: String?, + streamName: String, + namespace: String, streamSchema: JsonNode ): List { - val nameUpdater = getFieldNameUpdater(streamName!!, namespace, streamSchema) + val nameUpdater = getFieldNameUpdater(streamName, namespace, streamSchema) val objectSummaries = getAllSyncedObjects(streamName, namespace) val jsonRecords: MutableList = LinkedList() - for (objectSummary in objectSummaries!!) { - val `object` = s3Client!!.getObject(objectSummary!!.bucketName, objectSummary.key) + for (objectSummary in objectSummaries) { + val `object` = s3Client!!.getObject(objectSummary.bucketName, objectSummary.key) val uri = URI(String.format("s3a://%s/%s", `object`.bucketName, `object`.key)) val path = Path(uri) val hadoopConfig = GcsParquetWriter.getHadoopConfig(config) @@ -73,14 +73,14 @@ abstract class GcsBaseParquetDestinationAcceptanceTest : @Throws(Exception::class) override fun retrieveDataTypesFromPersistedFiles( - streamName: String?, - namespace: String? + streamName: String, + namespace: String ): Map?> { val objectSummaries = getAllSyncedObjects(streamName, namespace) val resultDataTypes: MutableMap?> = HashMap() - for (objectSummary in objectSummaries!!) { - val `object` = s3Client!!.getObject(objectSummary!!.bucketName, objectSummary.key) + for (objectSummary in objectSummaries) { + val `object` = s3Client!!.getObject(objectSummary.bucketName, objectSummary.key) val uri = URI(String.format("s3a://%s/%s", `object`.bucketName, `object`.key)) val path = Path(uri) val hadoopConfig = getHadoopConfig(config) diff --git a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationAcceptanceTest.kt index 5c2d390fe635f..92a18d74d4613 100644 --- a/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/gcs-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/gcs/GcsDestinationAcceptanceTest.kt @@ -30,6 +30,7 @@ import org.joda.time.DateTime import org.joda.time.DateTimeZone import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test +import org.mockito.Mockito.mock import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -46,9 +47,10 @@ import org.slf4j.LoggerFactory abstract class GcsDestinationAcceptanceTest(protected val outputFormat: S3Format) : DestinationAcceptanceTest() { protected var configJson: JsonNode? = null - protected lateinit var config: GcsDestinationConfig - protected lateinit var s3Client: AmazonS3 - protected lateinit var nameTransformer: NamingConventionTransformer + // Not a big fan of those mocks(). Here to make spotbugs happy + protected var config: GcsDestinationConfig = mock() + protected var s3Client: AmazonS3 = mock() + protected var nameTransformer: NamingConventionTransformer = mock() protected var s3StorageOperations: S3StorageOperations? = null protected val baseConfigJson: JsonNode @@ -96,23 +98,23 @@ abstract class GcsDestinationAcceptanceTest(protected val outputFormat: S3Format /** Helper method to retrieve all synced objects inside the configured bucket path. */ protected fun getAllSyncedObjects( - streamName: String?, - namespace: String? + streamName: String, + namespace: String ): List { - val namespaceStr = nameTransformer!!.getNamespace(namespace!!) - val streamNameStr = nameTransformer!!.getIdentifier(streamName!!) + val namespaceStr = nameTransformer.getNamespace(namespace) + val streamNameStr = nameTransformer.getIdentifier(streamName) val outputPrefix = s3StorageOperations!!.getBucketObjectPath( namespaceStr, streamNameStr, DateTime.now(DateTimeZone.UTC), - config!!.pathFormat!! + config.pathFormat!! ) // the child folder contains a non-deterministic epoch timestamp, so use the parent folder val parentFolder = outputPrefix.substring(0, outputPrefix.lastIndexOf("/") + 1) val objectSummaries = s3Client - .listObjects(config!!.bucketName, parentFolder) + .listObjects(config.bucketName, parentFolder) .objectSummaries .stream() .filter { o: S3ObjectSummary -> o.key.contains("$streamNameStr/") } diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.kt index f2709ea343f4d..fc86bbd22d000 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.kt @@ -39,8 +39,8 @@ abstract class S3BaseAvroDestinationAcceptanceTest protected constructor() : @Throws(Exception::class) override fun retrieveRecords( testEnv: TestDestinationEnv?, - streamName: String?, - namespace: String?, + streamName: String, + namespace: String, streamSchema: JsonNode ): List { val nameUpdater = diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseCsvDestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseCsvDestinationAcceptanceTest.kt index 4849db0603349..36bfd5312de2d 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseCsvDestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseCsvDestinationAcceptanceTest.kt @@ -36,8 +36,8 @@ abstract class S3BaseCsvDestinationAcceptanceTest : S3DestinationAcceptanceTest( @Throws(IOException::class) override fun retrieveRecords( testEnv: TestDestinationEnv?, - streamName: String?, - namespace: String?, + streamName: String, + namespace: String, streamSchema: JsonNode ): List { val objectSummaries = getAllSyncedObjects(streamName, namespace) diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseJsonlDestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseJsonlDestinationAcceptanceTest.kt index 381e75113e629..f89e54c965dea 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseJsonlDestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseJsonlDestinationAcceptanceTest.kt @@ -35,8 +35,8 @@ abstract class S3BaseJsonlDestinationAcceptanceTest protected constructor() : @Throws(IOException::class) override fun retrieveRecords( testEnv: TestDestinationEnv?, - streamName: String?, - namespace: String?, + streamName: String, + namespace: String, streamSchema: JsonNode ): List { val objectSummaries = getAllSyncedObjects(streamName, namespace) diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.kt index 401e40a15061c..e999be20c74d6 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.kt @@ -29,8 +29,8 @@ abstract class S3BaseParquetDestinationAcceptanceTest protected constructor() : @Throws(IOException::class, URISyntaxException::class) override fun retrieveRecords( testEnv: TestDestinationEnv?, - streamName: String?, - namespace: String?, + streamName: String, + namespace: String, streamSchema: JsonNode ): List { val nameUpdater =