From 0eace0ea3f4edd8fd4027b77eea14c8261427f0a Mon Sep 17 00:00:00 2001 From: Stephane Geneix Date: Fri, 29 Mar 2024 10:50:54 -0700 Subject: [PATCH] enable spotbugs for s3-destinations submodule --- .../airbyte-cdk/s3-destinations/build.gradle | 3 --- .../destination/s3/avro/JsonSchemaTypeTest.kt | 4 +-- .../S3AvroParquetDestinationAcceptanceTest.kt | 6 ++--- .../s3/S3BaseAvroDestinationAcceptanceTest.kt | 10 ++++---- .../S3BaseParquetDestinationAcceptanceTest.kt | 20 +++++++-------- .../s3/S3DestinationAcceptanceTest.kt | 25 ++++++++++--------- 6 files changed, 33 insertions(+), 35 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle index ea31fe9cd0f154..b5c9120ca019b5 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle @@ -32,9 +32,6 @@ compileKotlin { } } -spotbugsTest.enabled = false -spotbugsTestFixtures.enabled = false - dependencies { implementation project(':airbyte-cdk:java:airbyte-cdk:dependencies') implementation project(':airbyte-cdk:java:airbyte-cdk:core') diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/kotlin/io/airbyte/cdk/integrations/destination/s3/avro/JsonSchemaTypeTest.kt b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/kotlin/io/airbyte/cdk/integrations/destination/s3/avro/JsonSchemaTypeTest.kt index fd34b6674e287e..9f3ef0cf9af69b 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/kotlin/io/airbyte/cdk/integrations/destination/s3/avro/JsonSchemaTypeTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/kotlin/io/airbyte/cdk/integrations/destination/s3/avro/JsonSchemaTypeTest.kt @@ -16,11 +16,11 @@ class JsonSchemaTypeTest { @ParameterizedTest @ArgumentsSource(JsonSchemaTypeProvider::class) fun testFromJsonSchemaType( - type: String?, + type: String, airbyteType: String?, expectedJsonSchemaType: JsonSchemaType? ) { - Assertions.assertEquals(expectedJsonSchemaType, fromJsonSchemaType(type!!, airbyteType)) + Assertions.assertEquals(expectedJsonSchemaType, fromJsonSchemaType(type, airbyteType)) } class JsonSchemaTypeProvider : ArgumentsProvider { diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.kt index d827b6c1cf048e..4fc931aae890ba 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.kt @@ -115,9 +115,9 @@ abstract class S3AvroParquetDestinationAcceptanceTest protected constructor(s3Fo @Throws(Exception::class) protected abstract fun retrieveDataTypesFromPersistedFiles( - streamName: String?, - namespace: String? - ): Map?> + streamName: String, + namespace: String + ): Map> protected fun getTypes(record: GenericData.Record): Map> { val fieldList = diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.kt index fc86bbd22d0006..4a5ef8d710c76d 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.kt @@ -44,7 +44,7 @@ abstract class S3BaseAvroDestinationAcceptanceTest protected constructor() : streamSchema: JsonNode ): List { val nameUpdater = - AvroRecordHelper.getFieldNameUpdater(streamName!!, namespace, streamSchema) + AvroRecordHelper.getFieldNameUpdater(streamName, namespace, streamSchema) val objectSummaries = getAllSyncedObjects(streamName, namespace) val jsonRecords: MutableList = LinkedList() @@ -75,11 +75,11 @@ abstract class S3BaseAvroDestinationAcceptanceTest protected constructor() : @Throws(Exception::class) override fun retrieveDataTypesFromPersistedFiles( - streamName: String?, - namespace: String? - ): Map?> { + streamName: String, + namespace: String + ): Map> { val objectSummaries = getAllSyncedObjects(streamName, namespace) - val resultDataTypes: MutableMap?> = HashMap() + val resultDataTypes: MutableMap> = HashMap() for (objectSummary in objectSummaries) { val `object` = s3Client!!.getObject(objectSummary.bucketName, objectSummary.key) diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.kt index e999be20c74d63..fbc2c45260eaf7 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.kt @@ -34,13 +34,13 @@ abstract class S3BaseParquetDestinationAcceptanceTest protected constructor() : streamSchema: JsonNode ): List { val nameUpdater = - AvroRecordHelper.getFieldNameUpdater(streamName!!, namespace, streamSchema) + AvroRecordHelper.getFieldNameUpdater(streamName, namespace, streamSchema) val objectSummaries = getAllSyncedObjects(streamName, namespace) val jsonRecords: MutableList = LinkedList() - for (objectSummary in objectSummaries!!) { - val `object` = s3Client!!.getObject(objectSummary!!.bucketName, objectSummary.key) + for (objectSummary in objectSummaries) { + val `object` = s3Client!!.getObject(objectSummary.bucketName, objectSummary.key) val uri = URI(String.format("s3a://%s/%s", `object`.bucketName, `object`.key)) val path = Path(uri) val hadoopConfig = S3ParquetWriter.getHadoopConfig(s3DestinationConfig) @@ -68,14 +68,14 @@ abstract class S3BaseParquetDestinationAcceptanceTest protected constructor() : @Throws(Exception::class) override fun retrieveDataTypesFromPersistedFiles( - streamName: String?, - namespace: String? - ): Map?> { + streamName: String, + namespace: String + ): Map> { val objectSummaries = getAllSyncedObjects(streamName, namespace) - val resultDataTypes: MutableMap?> = HashMap() + val resultDataTypes: MutableMap> = HashMap() - for (objectSummary in objectSummaries!!) { - val `object` = s3Client!!.getObject(objectSummary!!.bucketName, objectSummary.key) + for (objectSummary in objectSummaries) { + val `object` = s3Client!!.getObject(objectSummary.bucketName, objectSummary.key) val uri = URI(String.format("s3a://%s/%s", `object`.bucketName, `object`.key)) val path = Path(uri) val hadoopConfig = S3ParquetWriter.getHadoopConfig(s3DestinationConfig) @@ -87,7 +87,7 @@ abstract class S3BaseParquetDestinationAcceptanceTest protected constructor() : var record: GenericData.Record? while ((parquetReader.read().also { record = it }) != null) { val actualDataTypes = getTypes(record!!) - resultDataTypes.putAll(actualDataTypes!!) + resultDataTypes.putAll(actualDataTypes) } } } diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.kt b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.kt index b064d8002f453d..0e245a4473e1dc 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.kt +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/kotlin/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.kt @@ -23,6 +23,7 @@ import java.util.stream.Collectors import org.apache.commons.lang3.RandomStringUtils import org.joda.time.DateTime import org.joda.time.DateTimeZone +import org.mockito.Mockito.mock import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -40,9 +41,9 @@ abstract class S3DestinationAcceptanceTest protected constructor(protected val outputFormat: S3Format) : DestinationAcceptanceTest() { protected val secretFilePath: String = "secrets/config.json" protected var configJson: JsonNode? = null - protected lateinit var s3DestinationConfig: S3DestinationConfig + protected var s3DestinationConfig: S3DestinationConfig = mock() protected var s3Client: AmazonS3? = null - protected lateinit var s3nameTransformer: NamingConventionTransformer + protected var s3nameTransformer: NamingConventionTransformer = mock() protected var s3StorageOperations: S3StorageOperations? = null protected val baseConfigJson: JsonNode @@ -68,23 +69,23 @@ protected constructor(protected val outputFormat: S3Format) : DestinationAccepta /** Helper method to retrieve all synced objects inside the configured bucket path. */ protected fun getAllSyncedObjects( - streamName: String?, - namespace: String? + streamName: String, + namespace: String ): List { - val namespaceStr = s3nameTransformer!!.getNamespace(namespace!!) - val streamNameStr = s3nameTransformer!!.getIdentifier(streamName!!) + val namespaceStr = s3nameTransformer.getNamespace(namespace) + val streamNameStr = s3nameTransformer.getIdentifier(streamName) val outputPrefix = s3StorageOperations!!.getBucketObjectPath( namespaceStr, streamNameStr, DateTime.now(DateTimeZone.UTC), - s3DestinationConfig!!.pathFormat!! + s3DestinationConfig.pathFormat!! ) // the child folder contains a non-deterministic epoch timestamp, so use the parent folder val parentFolder = outputPrefix.substring(0, outputPrefix.lastIndexOf("/") + 1) val objectSummaries = s3Client!! - .listObjects(s3DestinationConfig!!.bucketName, parentFolder) + .listObjects(s3DestinationConfig.bucketName, parentFolder) .objectSummaries .stream() .filter { o: S3ObjectSummary -> o.key.contains("$streamNameStr/") } @@ -141,7 +142,7 @@ protected constructor(protected val outputFormat: S3Format) : DestinationAccepta val keysToDelete: MutableList = LinkedList() val objects = s3Client!! - .listObjects(s3DestinationConfig!!.bucketName, s3DestinationConfig!!.bucketPath) + .listObjects(s3DestinationConfig.bucketName, s3DestinationConfig.bucketPath) .objectSummaries for (`object` in objects) { keysToDelete.add(DeleteObjectsRequest.KeyVersion(`object`.key)) @@ -150,12 +151,12 @@ protected constructor(protected val outputFormat: S3Format) : DestinationAccepta if (keysToDelete.size > 0) { LOGGER.info( "Tearing down test bucket path: {}/{}", - s3DestinationConfig!!.bucketName, - s3DestinationConfig!!.bucketPath + s3DestinationConfig.bucketName, + s3DestinationConfig.bucketPath ) val result = s3Client!!.deleteObjects( - DeleteObjectsRequest(s3DestinationConfig!!.bucketName).withKeys(keysToDelete) + DeleteObjectsRequest(s3DestinationConfig.bucketName).withKeys(keysToDelete) ) LOGGER.info("Deleted {} file(s).", result.deletedObjects.size) }