From 0a114ad995252ae2bd3fa9d206fd895c5290bf29 Mon Sep 17 00:00:00 2001 From: "Malik, Junaid" Date: Wed, 1 May 2024 16:04:46 +0800 Subject: [PATCH] #1296 add follow-up tests / naming improvements - remove java tests suite that was added only to gauge Java friendliness of the API. --- .../filter/FilterColumnValueParser.scala | 10 +- .../finos/vuu/util/schema/SchemaMapper.scala | 8 +- .../vuu/util/schema/SchemaMapperJavaTest.java | 14 -- .../util/schema/TypeConverterJavaTest.java | 43 ----- .../vuu/util/schema/SchemaMapperTest.scala | 160 ++++++++++-------- 5 files changed, 102 insertions(+), 133 deletions(-) delete mode 100644 vuu/src/test/java/org/finos/vuu/util/schema/SchemaMapperJavaTest.java delete mode 100644 vuu/src/test/java/org/finos/vuu/util/schema/TypeConverterJavaTest.java diff --git a/plugin/ignite-plugin/src/main/scala/org/finos/vuu/feature/ignite/filter/FilterColumnValueParser.scala b/plugin/ignite-plugin/src/main/scala/org/finos/vuu/feature/ignite/filter/FilterColumnValueParser.scala index d2d7661b4..8df104c40 100644 --- a/plugin/ignite-plugin/src/main/scala/org/finos/vuu/feature/ignite/filter/FilterColumnValueParser.scala +++ b/plugin/ignite-plugin/src/main/scala/org/finos/vuu/feature/ignite/filter/FilterColumnValueParser.scala @@ -26,19 +26,19 @@ private class ColumnValueParser(private val mapper: SchemaMapper) extends Filter override def parse(columnName: String, columnValue: String): Either[ErrorMessage, ParsedResult[Any]] = { mapper.externalSchemaField(columnName) match { - case Some(f) => RawColumnValueParser(f).parse(columnValue).map(ParsedResult(f, _)) + case Some(f) => CoreParser(f).parse(columnValue).map(ParsedResult(f, _)) case None => Left(externalFieldNotFoundError(columnName)) } } override def parse(columnName: String, columnValues: List[String]): Either[ErrorMessage, ParsedResult[List[Any]]] = { mapper.externalSchemaField(columnName) match { - case Some(f) => parseValues(RawColumnValueParser(f), columnValues) + case Some(f) => parseValues(CoreParser(f), columnValues) case None => Left(externalFieldNotFoundError(columnName)) } } - private def parseValues(parser: RawColumnValueParser, + private def parseValues(parser: CoreParser, columnValues: List[String]): Either[ErrorMessage, ParsedResult[List[Any]]] = { val (errors, parsedValues) = columnValues.partitionMap(parser.parse) val combinedError = errors.mkString("\n") @@ -56,8 +56,8 @@ private class ColumnValueParser(private val mapper: SchemaMapper) extends Filter private def externalFieldNotFoundError(columnName: String): String = s"Failed to find mapped external field for column `$columnName`" - private case class RawColumnValueParser(field: SchemaField) { - val column: Column = mapper.tableColumn(field.name).get + private case class CoreParser(field: SchemaField) { + val column: Column = mapper.internalVuuColumn(field.name).get def parse(columnValue: String): Either[ErrorMessage, Any] = { parseStringToColumnDataType(columnValue).flatMap(convertColumnValueToExternalFieldType) diff --git a/vuu/src/main/scala/org/finos/vuu/util/schema/SchemaMapper.scala b/vuu/src/main/scala/org/finos/vuu/util/schema/SchemaMapper.scala index 6b40e7338..780c00d92 100644 --- a/vuu/src/main/scala/org/finos/vuu/util/schema/SchemaMapper.scala +++ b/vuu/src/main/scala/org/finos/vuu/util/schema/SchemaMapper.scala @@ -12,7 +12,7 @@ import scala.util.Try * and vice versa. * */ trait SchemaMapper { - def tableColumn(extFieldName: String): Option[Column] + def internalVuuColumn(extFieldName: String): Option[Column] def externalSchemaField(columnName: String): Option[SchemaField] def toMappedExternalFieldType(columnName: String, columnValue: Any): Option[Any] def toMappedInternalColumnType(extFieldName: String, extFieldValue: Any): Option[Any] @@ -31,7 +31,7 @@ private class SchemaMapperImpl(private val externalSchema: ExternalEntitySchema, private val externalFieldByColumnName: Map[String, SchemaField] = getExternalSchemaFieldsByColumnName private val internalColumnByExtFieldName: Map[String, Column] = getTableColumnByExternalField - override def tableColumn(extFieldName: String): Option[Column] = internalColumnByExtFieldName.get(extFieldName) + override def internalVuuColumn(extFieldName: String): Option[Column] = internalColumnByExtFieldName.get(extFieldName) override def externalSchemaField(columnName: String): Option[SchemaField] = externalFieldByColumnName.get(columnName) override def toInternalRowMap(externalValues: List[_]): Map[String, Any] = toInternalRowMap(externalValues.toArray) @@ -47,13 +47,13 @@ private class SchemaMapperImpl(private val externalSchema: ExternalEntitySchema, override def toMappedExternalFieldType(columnName: String, columnValue: Any): Option[Any] = { externalSchemaField(columnName).flatMap(field => { - val col = tableColumn(field.name).get + val col = internalVuuColumn(field.name).get safeTypeConvert(columnValue, castToAny(col.dataType), field.dataType) }) } override def toMappedInternalColumnType(extFieldName: String, extFieldValue: Any): Option[Any] = { - tableColumn(extFieldName).flatMap(col => { + internalVuuColumn(extFieldName).flatMap(col => { val field = externalSchemaField(col.name).get safeTypeConvert(extFieldValue, castToAny(field.dataType), col.dataType) }) diff --git a/vuu/src/test/java/org/finos/vuu/util/schema/SchemaMapperJavaTest.java b/vuu/src/test/java/org/finos/vuu/util/schema/SchemaMapperJavaTest.java deleted file mode 100644 index 018ff106d..000000000 --- a/vuu/src/test/java/org/finos/vuu/util/schema/SchemaMapperJavaTest.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.finos.vuu.util.schema; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -public class SchemaMapperJavaTest { - - @Test - public void test() { -// TypeConverter typeConverter = Integer::parseInt; -// assertEquals(Integer.valueOf(20), typeConverter.apply("20")); - } -} diff --git a/vuu/src/test/java/org/finos/vuu/util/schema/TypeConverterJavaTest.java b/vuu/src/test/java/org/finos/vuu/util/schema/TypeConverterJavaTest.java deleted file mode 100644 index 6ef28211e..000000000 --- a/vuu/src/test/java/org/finos/vuu/util/schema/TypeConverterJavaTest.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.finos.vuu.util.schema; - -import org.finos.vuu.util.types.TypeConverter; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -public class TypeConverterJavaTest { - - @Test - public void test_quick_instantiation_through_TypeConverter_apply() { - final var tc = TypeConverter.apply(String.class, Integer.class, Integer::parseInt); - - assertEquals(Integer.valueOf(20), tc.convert("20")); - assertEquals(tc.name(), "java.lang.String->java.lang.Integer"); - } - - @Test - public void test_instantiation_through_interface_implementation() { - class MyTypeConverter implements TypeConverter { - - @Override - public Class fromClass() { - return String.class; - } - - @Override - public Class toClass() { - return double.class; - } - - @Override - public Double convert(String v) { - return Double.valueOf(v); - } - } - - final var tc = new MyTypeConverter(); - - assertEquals(Double.valueOf(20.56), tc.convert("20.56")); - assertEquals(tc.name(), "java.lang.String->java.lang.Double"); - } -} diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperTest.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperTest.scala index e838f82c9..26e299f8d 100644 --- a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperTest.scala +++ b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperTest.scala @@ -1,9 +1,8 @@ package org.finos.vuu.util.schema -import org.finos.vuu.core.module.vui.VuiStateModule.stringToFieldDef -import org.finos.vuu.core.table.{Column, Columns, SimpleColumn} +import org.finos.vuu.core.table.{Column, SimpleColumn} import org.finos.vuu.util.schema.SchemaMapper.InvalidSchemaMapException -import org.finos.vuu.util.schema.SchemaMapperTest.{externalFields, externalSchema, fieldsMap, fieldsMapWithoutAssetClass, internalColumns} +import org.finos.vuu.util.schema.SchemaMapperTest.{column, externalSchema, fieldsMap, fieldsMapWithoutAssetClass, givenColumns, givenExternalSchema, internalColumns, schemaField} import org.finos.vuu.util.types.{TypeConverter, TypeConverterContainerBuilder} import org.scalatest.featurespec.AnyFeatureSpec import org.scalatest.matchers.should.Matchers @@ -80,36 +79,36 @@ class SchemaMapperTest extends AnyFeatureSpec with Matchers { field.get shouldEqual SchemaField("externalRic", classOf[String], 1) } - Scenario("returns None if column not present in the mapped fields") { + Scenario("returns None if column not present in mapped fields") { val field = mapper.externalSchemaField("assetClass") field shouldEqual None } } - Feature("tableColumn") { + Feature("internalVuuColumn") { val mapper = SchemaMapperBuilder(externalSchema, internalColumns) .withFieldsMap(fieldsMapWithoutAssetClass) .build() Scenario("can get internal column from external field name") { - val column = mapper.tableColumn("externalRic") + val column = mapper.internalVuuColumn("externalRic") column.get shouldEqual SimpleColumn("ric", 1, classOf[String]) } - Scenario("returns None if external field not present in the mapped fields") { - val column = mapper.tableColumn("assetClass") + Scenario("returns None if external field not present in mapped fields") { + val column = mapper.internalVuuColumn("assetClass") column shouldEqual None } } Feature("toMappedExternalFieldType") { - val bigDecimalSchemaField = SchemaField("bigDecimalPrice", classOf[BigDecimal], 0) - val doubleColumn = SimpleColumn("doublePrice", 0, classOf[Double]) + val bigDecimalSchemaField = schemaField("bigDecimalPrice", classOf[BigDecimal], 0) + val doubleColumn = column("doublePrice", classOf[Double], 0) val tcContainer = TypeConverterContainerBuilder().withoutDefaults() .withConverter(TypeConverter[BigDecimal, Double](classOf[BigDecimal], classOf[Double], _.doubleValue())) .withConverter(TypeConverter[Double, BigDecimal](classOf[Double], classOf[BigDecimal], new BigDecimal(_))) .build() - val schemaMapper = SchemaMapperBuilder(TestEntitySchema(List(bigDecimalSchemaField)), Array(doubleColumn)) + val schemaMapper = SchemaMapperBuilder(givenExternalSchema(bigDecimalSchemaField), givenColumns(doubleColumn)) .withFieldsMap(Map("bigDecimalPrice" -> "doublePrice")) .withTypeConverters(tcContainer) .build() @@ -131,13 +130,13 @@ class SchemaMapperTest extends AnyFeatureSpec with Matchers { } Feature("toMappedInternalColumnType") { - val bigDecimalSchemaField = SchemaField("bigDecimalPrice", classOf[BigDecimal], 0) - val doubleColumn = SimpleColumn("doublePrice", 0, classOf[Double]) + val bigDecimalSchemaField = schemaField("bigDecimalPrice", classOf[BigDecimal], 0) + val doubleColumn = column("doublePrice", classOf[Double], 0) val tcContainer = TypeConverterContainerBuilder().withoutDefaults() .withConverter(TypeConverter[BigDecimal, Double](classOf[BigDecimal], classOf[Double], _.doubleValue())) .withConverter(TypeConverter[Double, BigDecimal](classOf[Double], classOf[BigDecimal], new BigDecimal(_))) .build() - val schemaMapper = SchemaMapperBuilder(TestEntitySchema(List(bigDecimalSchemaField)), Array(doubleColumn)) + val schemaMapper = SchemaMapperBuilder(givenExternalSchema(bigDecimalSchemaField), givenColumns(doubleColumn)) .withFieldsMap(Map("bigDecimalPrice" -> "doublePrice")) .withTypeConverters(tcContainer) .build() @@ -160,76 +159,98 @@ class SchemaMapperTest extends AnyFeatureSpec with Matchers { Feature("validation on instantiation") { Scenario("fails when mapped external field not found in external schema") { + val externalSchema = givenExternalSchema(schemaField("externalId")) + val columns = givenColumns(column("ric")) + val exception = intercept[InvalidSchemaMapException]( - SchemaMapperBuilder(externalSchema, Columns.fromNames("ric".int())) - .withFieldsMap(Map("non-existent" -> "ric")).build() + SchemaMapperBuilder(externalSchema, columns).withFieldsMap(Map("externalRic" -> "ric")).build() ) + exception shouldBe a[RuntimeException] - exception.getMessage should include regex s"[Ff]ield `non-existent` not found" + exception.getMessage should include regex s"[Ff]ield `externalRic` not found" } Scenario("fails when mapped internal field not found in internal columns") { + val externalSchema = givenExternalSchema(schemaField("externalId")) + val columns = givenColumns(column("ric")) + val exception = intercept[InvalidSchemaMapException]( - SchemaMapperBuilder(externalSchema, Columns.fromNames("id".int())) - .withFieldsMap(Map("externalId" -> "absent-col")).build() + SchemaMapperBuilder(externalSchema, columns).withFieldsMap(Map("externalId" -> "id")).build() ) + exception shouldBe a[RuntimeException] - exception.getMessage should include regex "[Cc]olumn `absent-col` not found" + exception.getMessage should include regex "[Cc]olumn `id` not found" } Scenario("fails when external->internal map contains duplicated internal fields") { + val externalSchema = givenExternalSchema(schemaField("externalId"), schemaField("parentId")) + val columns = givenColumns(column("id")) + val exception = intercept[InvalidSchemaMapException]( - SchemaMapperBuilder(externalSchema, Columns.fromNames("id".int(), "ric".string())) - .withFieldsMap(Map("externalId" -> "id", "externalRic" -> "id")) - .build() + SchemaMapperBuilder(externalSchema, columns).withFieldsMap(Map("externalId" -> "id", "parentId" -> "id")).build() ) + exception shouldBe a[RuntimeException] exception.getMessage should include("duplicated column names") } Scenario("fails when types differ b/w mapped fields and type converter is not provided") { + val externalSchema = givenExternalSchema(schemaField("externalId", classOf[Long], index = 0)) + val columns = givenColumns(column("id", classOf[String], index = 0)) val emptyTypeConverterContainer = TypeConverterContainerBuilder().withoutDefaults().build() + val exception = intercept[InvalidSchemaMapException]( - SchemaMapperBuilder(externalSchema, internalColumns) - .withFieldsMap(fieldsMap) + SchemaMapperBuilder(externalSchema, columns) + .withFieldsMap(Map("externalId" -> "id")) .withTypeConverters(emptyTypeConverterContainer) .build() ) + exception.getMessage should include regex ".*TypeConverter.* not found.*" - exception.message should include("[ java.lang.Double->java.lang.String ]") - exception.message should include("[ java.lang.String->java.lang.Double ]") + exception.message should include("[ java.lang.Long->java.lang.String ]") + exception.message should include("[ java.lang.String->java.lang.Long ]") } } Feature("Build schema mapper without user-defined fields map") { Scenario("can generate mapper with exact fields matched by index") { - val mapper = SchemaMapperBuilder(externalSchema, internalColumns).build() + val externalSchema = givenExternalSchema(schemaField("externalId", index = 0), schemaField("priceExt", index = 1)) + val columns = givenColumns(column("id", index = 0), column("price", index = 1)) - mapper.tableColumn("externalId").get.name shouldEqual "id" - mapper.tableColumn("externalRic").get.name shouldEqual "ric" - mapper.tableColumn("assetClass").get.name shouldEqual "assetClass" - mapper.tableColumn("price").get.name shouldEqual "price" - mapper.tableColumn("side").get.name shouldEqual "side" + val mapper = SchemaMapperBuilder(externalSchema, columns).build() + + mapper.internalVuuColumn("externalId").get.name shouldEqual "id" + mapper.internalVuuColumn("priceExt").get.name shouldEqual "price" + } + + Scenario("can generate mapper with exact fields even when the fields are not ordered by their index") { + val externalSchema = givenExternalSchema(schemaField("priceExt", index = 1), schemaField("externalId", index = 0)) + val columns = givenColumns(column("id", index = 0), column("price", index = 1)) + + val mapper = SchemaMapperBuilder(externalSchema, columns).build() + + mapper.internalVuuColumn("externalId").get.name shouldEqual "id" + mapper.internalVuuColumn("priceExt").get.name shouldEqual "price" } Scenario("can generate mapper when an external field has no matched column") { - val mapper = SchemaMapperBuilder(externalSchema, internalColumns.slice(0, 3)).build() + val externalSchema = givenExternalSchema(schemaField("priceExt", index = 1), schemaField("externalId", index = 0)) + val columns = givenColumns(column("id", index = 0)) + + val mapper = SchemaMapperBuilder(externalSchema, columns).build() - mapper.tableColumn("externalId") shouldBe empty - mapper.tableColumn("externalRic").get.name shouldEqual "ric" - mapper.tableColumn("assetClass").get.name shouldEqual "assetClass" - mapper.tableColumn("price").get.name shouldEqual "price" - mapper.tableColumn("side") shouldBe empty + mapper.internalVuuColumn("externalId").get.name shouldEqual "id" + mapper.internalVuuColumn("priceExt") shouldBe empty } Scenario("can generate mapper when a column has no matched external field") { - val mapper = SchemaMapperBuilder(TestEntitySchema(externalFields.slice(0, 3)), internalColumns).build() + val externalSchema = givenExternalSchema(schemaField("priceExt", index = 1)) + val columns = givenColumns(column("id", index = 0), column("price", index = 1)) - mapper.tableColumn("externalId").get.name shouldEqual "id" - mapper.tableColumn("externalRic") shouldBe empty - mapper.tableColumn("assetClass").get.name shouldBe "assetClass" - mapper.tableColumn("price").get.name shouldEqual "price" - mapper.tableColumn("side") shouldBe empty + val mapper = SchemaMapperBuilder(externalSchema, columns).build() + + mapper.externalSchemaField("price").get.name shouldEqual "priceExt" + mapper.externalSchemaField("id") shouldBe empty } } } @@ -240,30 +261,35 @@ private case class TestDto(externalId: Int, externalRic: String, assetClass: Str private object SchemaMapperTest { - // no need to be sorted by their index - val externalFields: List[SchemaField] = List( - SchemaField("externalId", classOf[Int], 0), - SchemaField("assetClass", classOf[String], 2), - SchemaField("price", classOf[String], 3), - SchemaField("externalRic", classOf[String], 1), - SchemaField("side", classOf[java.lang.Character], 4), + private def givenExternalSchema(fields: SchemaField*): ExternalEntitySchema = TestEntitySchema(fields.toList) + + private def schemaField(name: String, dataType: Class[_] = classOf[Any], index: Int = -1): SchemaField = + SchemaField(name, dataType, index) + + private def givenColumns(columns: Column*): Array[Column] = columns.toArray + + private def column(name: String, dataType: Class[_] = classOf[Any], index: Int = -1): Column = + SimpleColumn(name, index, dataType) + + private val externalFields: List[SchemaField] = List( + schemaField("externalId", classOf[Int], 0), + schemaField("externalRic", classOf[String], 1), + schemaField("assetClass", classOf[String], 2), + schemaField("price", classOf[String], 3), + schemaField("side", classOf[java.lang.Character], 4), ) - val externalSchema: TestEntitySchema = TestEntitySchema(externalFields) - - val internalColumns: Array[Column] = { - val columns = Columns.fromNames( - "id".int(), - "ric".string(), - "assetClass".string(), - "price".double(), - "side".char(), - ) - // no need to be sorted by their index - columns.tail.appended(columns.head) - } + private val externalSchema: ExternalEntitySchema = givenExternalSchema(externalFields: _*) + + private val internalColumns: Array[Column] = givenColumns( + column("id", classOf[Int], 0), + column("ric", classOf[String], 1), + column("assetClass", classOf[String], 2), + column("price", classOf[Double], 3), + column("side", classOf[Char], 4), + ) - val fieldsMap: Map[String, String] = Map( + private val fieldsMap: Map[String, String] = Map( "externalId" -> "id", "externalRic" -> "ric", "price" -> "price", @@ -271,5 +297,5 @@ private object SchemaMapperTest { "side" -> "side", ) - val fieldsMapWithoutAssetClass: Map[String, String] = fieldsMap.filter({ case (k, _) => k != "assetClass"}) + private val fieldsMapWithoutAssetClass: Map[String, String] = fieldsMap.filter({ case (k, _) => k != "assetClass"}) }