finos#1296 in-progress

plugin/ignite-plugin/src/main/scala/org/finos/vuu/feature/ignite/filter/IgniteSqlFilterClause.scala

@@ -28,14 +28,16 @@ case class AndIgniteSqlFilterClause(clauses:List[IgniteSqlFilterClause]) extends
 }
 
 case class EqIgniteSqlFilterClause(columnName: String, value: String) extends IgniteSqlFilterClause {
-  override def toSql(schemaMapper: SchemaMapper): String =
-    schemaMapper.externalSchemaField(columnName) match {
-      case Some(f) => f.dataType match {
-        case CharDataType | StringDataType => eqSql(f.name, quotedString(value))
-        case _ => eqSql(f.name, value)
-      }
+  override def toSql(schemaMapper: SchemaMapper): String = {
+    SqlFilterColumnValueParser(schemaMapper).parseColumnValue(columnName, value) match {
+      case Some((extField, extValue)) =>
+        extField.dataType match {
+          case CharDataType | StringDataType => eqSql(extField.name, quotedString(extValue))
+          case _ => eqSql(extField.name, extValue)
+        }
       case None => logMappingErrorAndReturnEmptySql(columnName)
     }
+  }
 
   private def eqSql(field: String, processedVal: String): String = {
     s"$field = $processedVal"
@@ -44,7 +46,7 @@ case class EqIgniteSqlFilterClause(columnName: String, value: String) extends Ig
 
 case class NeqIgniteSqlFilterClause(columnName: String, value: String) extends IgniteSqlFilterClause {
   override def toSql(schemaMapper: SchemaMapper): String =
-    schemaMapper.externalSchemaField(columnName) match {
+    schemaMapper.getMappedExternalField(columnName) match {
       case Some(field) => field.dataType match {
         case CharDataType | StringDataType => neqSql(field.name, quotedString(value))
         case _ => neqSql(field.name, value)
@@ -60,7 +62,7 @@ case class NeqIgniteSqlFilterClause(columnName: String, value: String) extends I
 //todo why is number cast double? need to cast back to original type?
 case class RangeIgniteSqlFilterClause(op: RangeOp)(columnName: String, value: Double) extends IgniteSqlFilterClause {
   override def toSql(schemaMapper: SchemaMapper): String =
-    schemaMapper.externalSchemaField(columnName) match {
+    schemaMapper.getMappedExternalField(columnName) match {
       case Some(f) => s"${f.name} ${op.value} $value"
       case None    => logMappingErrorAndReturnEmptySql(columnName)
     }
@@ -69,7 +71,7 @@ case class RangeIgniteSqlFilterClause(op: RangeOp)(columnName: String, value: Do
 
 case class StartsIgniteSqlFilterClause(columnName: String, value: String) extends IgniteSqlFilterClause with StrictLogging {
   override def toSql(schemaMapper: SchemaMapper): String = {
-    schemaMapper.externalSchemaField(columnName) match {
+    schemaMapper.getMappedExternalField(columnName) match {
       case Some(f) => f.dataType match {
         case StringDataType => s"${f.name} LIKE '$value%'"
         case _ => logErrorAndReturnEmptySql(s"`Starts` clause unsupported for non string column: `${f.name}` (${f.dataType})")
@@ -81,7 +83,7 @@ case class StartsIgniteSqlFilterClause(columnName: String, value: String) extend
 
 case class EndsIgniteSqlFilterClause(columnName: String, value: String) extends IgniteSqlFilterClause with StrictLogging {
   override def toSql(schemaMapper: SchemaMapper): String =
-    schemaMapper.externalSchemaField(columnName) match {
+    schemaMapper.getMappedExternalField(columnName) match {
       case Some(f) => f.dataType match {
         case StringDataType => s"${f.name} LIKE '%$value'"
         case _ => logErrorAndReturnEmptySql(s"`Ends` clause unsupported for non string column: `${f.name}` (${f.dataType})")
@@ -92,7 +94,7 @@ case class EndsIgniteSqlFilterClause(columnName: String, value: String) extends
 
 case class InIgniteSqlFilterClause(columnName: String, values: List[String]) extends IgniteSqlFilterClause with StrictLogging {
   override def toSql(schemaMapper: SchemaMapper): String =
-    schemaMapper.externalSchemaField(columnName) match {
+    schemaMapper.getMappedExternalField(columnName) match {
       case Some(f) => f.dataType match {
         case CharDataType | StringDataType => inQuery(f.name, values.map(quotedString(_)))
         case _ => inQuery(f.name, values)

plugin/ignite-plugin/src/main/scala/org/finos/vuu/feature/ignite/filter/SqlFilterColumnValueParser.scala

@@ -0,0 +1,47 @@
+package org.finos.vuu.feature.ignite.filter
+
+import com.typesafe.scalalogging.StrictLogging
+import org.finos.vuu.util.schema.typeConversion.TypeConverter.buildConverterName
+import org.finos.vuu.util.schema.{SchemaField, SchemaMapper}
+
+trait SqlFilterColumnValueParser {
+  def parseColumnValue(columnName: String, columnValue: String): Option[(SchemaField, String)]
+  def parseColumnValues(columnName: String, columnValues: List[String]): Option[(SchemaField, List[String])]
+}
+
+object SqlFilterColumnValueParser {
+  def apply(schemaMapper: SchemaMapper): SqlFilterColumnValueParser = new ColumnValueParser(schemaMapper)
+}
+
+private class ColumnValueParser(private val mapper: SchemaMapper) extends SqlFilterColumnValueParser with StrictLogging {
+
+  override def parseColumnValue(columnName: String, columnValue: String): Option[(SchemaField, String)] = {
+    mapper.getMappedExternalField(columnName).flatMap(f => parseValue(f, columnName, columnValue).map((f, _)))
+  }
+
+  override def parseColumnValues(columnName: String, columnValues: List[String]): Option[(SchemaField, List[String])] = {
+    mapper.getMappedExternalField(columnName) match {
+      case Some(f) => Some(f, columnValues.flatMap(v => parseValue(f, columnName, v)))
+      case None    => None
+    }
+  }
+
+  private def parseValue(field: SchemaField, columnName: String, columnValue: String): Option[String] = {
+    def useDefaultToString = logErrorAndUseDefaultToString(field)
+
+    mapper.parseStringToColumnType(columnValue, columnName)
+      .flatMap(mapper.convertColumnValueToMappedExternalType(_, columnName))
+      .map(v => mapper.convertExternalValueToString(v, field.name).getOrElse(useDefaultToString(v)))
+  }
+
+  private def logErrorAndUseDefaultToString(field: SchemaField): Any => String = {
+    extValue =>
+      logger.warn(
+        s"Could not find a converter [${buildConverterName(field.dataType, classOf[String])}] " +
+          s"required for SQL filters to be applied to ${field.name}. Falling back to using the " +
+          s"default `toString`."
+      )
+      Option(extValue).map(_.toString).orNull
+  }
+
+}

plugin/ignite-plugin/src/main/scala/org/finos/vuu/feature/ignite/sort/IgniteSqlSortBuilder.scala

@@ -19,7 +19,7 @@ class IgniteSqlSortBuilder {
   private def toSortString(columnName: String,
                            sortDirection: SortDirection.TYPE,
                            schemaMapper: SchemaMapper): Option[String] = {
-    schemaMapper.externalSchemaField(columnName) match {
+    schemaMapper.getMappedExternalField(columnName) match {
       case Some(f) => Some(s"${f.name} ${toSQL(sortDirection)}")
       case None => None
     }

plugin/ignite-plugin/src/test/scala/org/finos/vuu/feature/ignite/sort/IgniteSqlSortBuilderTest.scala

@@ -12,7 +12,7 @@ class IgniteSqlSortBuilderTest extends AnyFeatureSpec with MockFactory {
 
   Feature("IgniteSqlSortBuilder") {
     Scenario("can create sql order by clause for ignite column with different name") {
-      (schemaMapper.externalSchemaField _).expects("parentOrderId").returns(Option(testSchemaField("orderId")))
+      (schemaMapper.getMappedExternalField _).expects("parentOrderId").returns(Option(testSchemaField("orderId")))
 
       val sortSpecInternal = Map("parentOrderId"-> SortDirection.Descending)
       val sortSql = sortBuilder.toSql(sortSpecInternal, schemaMapper)
@@ -21,8 +21,8 @@ class IgniteSqlSortBuilderTest extends AnyFeatureSpec with MockFactory {
     }
 
     Scenario("can create sql order by clause for multiple ignite columns") {
-      (schemaMapper.externalSchemaField _).expects("column1").returns(Option(testSchemaField("column1")))
-      (schemaMapper.externalSchemaField _).expects("column2").returns(Option(testSchemaField("column2")))
+      (schemaMapper.getMappedExternalField _).expects("column1").returns(Option(testSchemaField("column1")))
+      (schemaMapper.getMappedExternalField _).expects("column2").returns(Option(testSchemaField("column2")))
 
       val sortSpecInternal = Map(
         "column1" -> SortDirection.Descending,
@@ -34,7 +34,7 @@ class IgniteSqlSortBuilderTest extends AnyFeatureSpec with MockFactory {
     }
 
     Scenario("skip sort if no mapping found to ignite columns") {
-      (schemaMapper.externalSchemaField _).expects("someTableColumnNotInMap").returns(None)
+      (schemaMapper.getMappedExternalField _).expects("someTableColumnNotInMap").returns(None)
 
       val sortSpecInternal = Map("someTableColumnNotInMap" -> SortDirection.Descending)
       val sortSql = sortBuilder.toSql(sortSpecInternal, schemaMapper)

plugin/ignite-plugin/src/test/scala/org/finos/vuu/feature/ignite/sort/IgniteSqlSortingTest.scala

@@ -125,7 +125,7 @@ class IgniteSqlSortingTest extends IgniteTestsBase with MockFactory {
   private def mockSchemaMapper(fields: Iterable[String]): SchemaMapper = {
     val schemaMapper = stub[SchemaMapper]
     fields.foreach (field =>
-      (schemaMapper.externalSchemaField _).when(field).returns(Option(SchemaField(field, classOf[Any], -1)))
+      (schemaMapper.getMappedExternalField _).when(field).returns(Option(SchemaField(field, classOf[Any], -1)))
     )
     schemaMapper
   }

vuu/src/main/scala/org/finos/vuu/util/schema/SchemaMapper.scala

@@ -13,8 +13,14 @@ import org.finos.vuu.util.schema.typeConversion.{TypeConverter, TypeConverterCon
  * future.
  * */
 trait SchemaMapper {
-  def tableColumn(extFieldName: String): Option[Column]
-  def externalSchemaField(columnName: String): Option[SchemaField]
+  def getColumn(name: String): Option[Column]
+  def getMappedColumn(extFieldName: String): Option[Column]
+  def getExternalField(name: String): Option[SchemaField]
+  def getMappedExternalField(columnName: String): Option[SchemaField]
+  def convertColumnValueToMappedExternalType(columnValue: Any, columnName: String): Option[Any]
+  def convertExternalValueToMappedColumnType(extFieldValue: Any, extFieldName: String): Option[Any]
+  def convertExternalValueToString(externalValue: Any, extFieldName: String): Option[String]
+  def parseStringToColumnType(columnValue: String, columnName: String): Option[Any]
   def toInternalRowMap(externalValues: List[_]): Map[String, Any]
   def toInternalRowMap(externalDto: Product): Map[String, Any]
 }
@@ -133,22 +139,65 @@ private class SchemaMapperImpl(private val externalSchema: ExternalEntitySchema,
                                private val typeConverterContainer: TypeConverterContainer) extends SchemaMapper {
   private val externalFieldByColumnName: Map[String, SchemaField] = getExternalSchemaFieldsByColumnName
   private val internalColumnByExtFieldName: Map[String, Column] = getTableColumnByExternalField
+  private val columnsMap: Map[String, Column] = internalColumns.map(c => (c.name, c)).toMap
+  private val extFieldsMap: Map[String, SchemaField] = externalSchema.fields.map(f => (f.name, f)).toMap
+
+  override def getColumn(name: String): Option[Column] = columnsMap.get(name)
+  override def getExternalField(name: String): Option[SchemaField] = extFieldsMap.get(name)
+  override def getMappedColumn(extFieldName: String): Option[Column] = internalColumnByExtFieldName.get(extFieldName)
+  override def getMappedExternalField(columnName: String): Option[SchemaField] = externalFieldByColumnName.get(columnName)
 
-  override def tableColumn(extFieldName: String): Option[Column] = internalColumnByExtFieldName.get(extFieldName)
-  override def externalSchemaField(columnName: String): Option[SchemaField] = externalFieldByColumnName.get(columnName)
   override def toInternalRowMap(externalValues: List[_]): Map[String, Any] = toInternalRowMap(externalValues.toArray)
   override def toInternalRowMap(externalDto: Product): Map[String, Any] = toInternalRowMap(externalDto.productIterator.toArray)
-
   private def toInternalRowMap(externalValues: Array[_]): Map[String, Any] = {
-    externalFieldByColumnName.map({ case (_, extField) =>
+    externalFieldByColumnName.map({ case (columnName, extField) =>
       val extFieldValue = externalValues(extField.index)
-      val extFieldDataType = extField.dataType.asInstanceOf[Class[Any]]
-      val column = tableColumn(extField.name).get
-      val columnValue = typeConverterContainer.convert(extFieldValue, extFieldDataType, column.dataType).get
-      (column.name, columnValue)
+      val columnValue = convertExternalValueToMappedColumnType(extFieldValue, extField.name).get
+      (columnName, columnValue)
+    })
+  }
+
+//  /**
+//   * Parses `columnValue` to its original type and then converts the parsed
+//   * column value to mapped external field's value type using the passed
+//   * type converter.
+//   *
+//   * @param columnValue represents an internal Vuu column's value as string
+//   * @param columnName internal Vuu column's name
+//   */
+//  override def parseToExternalFieldType(columnValue: String, columnName: String): Option[Any] = {
+//    val parsedColumnValue = parseStringToColumnType(columnValue, columnName)
+//
+//    getMappedExternalField(columnName).map(field => {
+//      val col = getMappedColumn(field.name).get
+//      typeConverterContainer.convert(parsedColumnValue, col.dataType.asInstanceOf[Class[Any]], field.dataType)
+//    })
+//  }
+
+  def convertColumnValueToMappedExternalType(columnValue: Any, columnName: String): Option[Any] = {
+    getMappedExternalField(columnName).map(field => {
+      val col = getMappedColumn(field.name).get
+      typeConverterContainer.convert(columnValue, col.dataType.asInstanceOf[Class[Any]], field.dataType)
+    })
+  }
+
+  def convertExternalValueToMappedColumnType(extFieldValue: Any, extFieldName: String): Option[Any] = {
+    getMappedColumn(extFieldName).map(col => {
+      val field = getMappedExternalField(col.name).get
+      typeConverterContainer.convert(extFieldValue, field.dataType.asInstanceOf[Class[Any]], col.dataType)
     })
   }
 
+  def parseStringToColumnType(columnValue: String, columnName: String): Option[Any] = {
+    // col.dataType needs to be restricted so that we have all default type converters covered
+    getColumn(columnName).flatMap(col => typeConverterContainer.convert(columnValue, classOf[String], col.dataType))
+  }
+
+  override def convertExternalValueToString(externalValue: Any, extFieldName: String): Option[String] = {
+    getExternalField(extFieldName)
+      .flatMap(f => typeConverterContainer.convert(externalValue, f.dataType.asInstanceOf[Class[Any]], classOf[String]))
+  }
+
   private def getExternalSchemaFieldsByColumnName =
     externalSchema.fields.flatMap(f =>
       Option.when(columnNameByExternalField.contains(f.name))(columnNameByExternalField(f.name), f)

vuu/src/main/scala/org/finos/vuu/util/schema/typeConversion/DefaultTypeConverters.scala

@@ -6,9 +6,13 @@ object DefaultTypeConverters {
   val stringToDoubleConverter: TypeConverter[String, Double] = TypeConverter(classOf[String], classOf[Double], withNullSafety[String, Double](_, _.toDouble))
   val stringToLongConverter: TypeConverter[String, Long] = TypeConverter[String, Long](classOf[String], classOf[Long], withNullSafety[String, Long](_, _.toLong))
   val stringToIntConverter: TypeConverter[String, Integer] = TypeConverter(classOf[String], classOf[Integer], withNullSafety[String, Integer](_, _.toInt))
+
+  val shortToStringConverter: TypeConverter[Short, String] = TypeConverter(classOf[Short], classOf[String], withNullSafety[Short, String](_, _.toString))
   val intToStringConverter: TypeConverter[Integer, String] = TypeConverter(classOf[Integer], classOf[String], withNullSafety[Integer, String](_, _.toString))
   val longToStringConverter: TypeConverter[Long, String] = TypeConverter(classOf[Long], classOf[String], withNullSafety[Long, String](_, _.toString))
+  val floatToStringConverter: TypeConverter[Float, String] = TypeConverter(classOf[Float], classOf[String], withNullSafety[Float, String](_, _.toString))
   val doubleToStringConverter: TypeConverter[Double, String] = TypeConverter(classOf[Double], classOf[String], withNullSafety[Double, String](_, _.toString))
+//  val bigDecimalToStringConverter:
 
 
   private def withNullSafety[T1, T2 >: Null](v: T1, fn: T1 => T2): T2 = Option(v).map(fn).orNull

vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperTest.scala

@@ -65,12 +65,12 @@ class SchemaMapperTest extends AnyFeatureSpec with Matchers {
     val mapper = SchemaMapper(externalSchema, internalColumns, fieldsMapWithoutAssetClass)
 
     Scenario("can get external schema field from internal column name") {
-      val field = mapper.externalSchemaField("ric")
+      val field = mapper.getMappedExternalField("ric")
       field.get shouldEqual SchemaField("externalRic", classOf[String], 1)
     }
 
     Scenario("returns None if column not present in the mapped fields") {
-      val field = mapper.externalSchemaField("assetClass")
+      val field = mapper.getMappedExternalField("assetClass")
       field shouldEqual None
     }
   }
@@ -79,12 +79,12 @@ class SchemaMapperTest extends AnyFeatureSpec with Matchers {
     val mapper = SchemaMapper(externalSchema, internalColumns, fieldsMapWithoutAssetClass)
 
     Scenario("can get internal column from external field name") {
-      val column = mapper.tableColumn("externalRic")
+      val column = mapper.getMappedColumn("externalRic")
       column.get shouldEqual SimpleColumn("ric", 1, classOf[String])
     }
 
     Scenario("returns None if external field not present in the mapped fields") {
-      val column = mapper.tableColumn("assetClass")
+      val column = mapper.getMappedColumn("assetClass")
       column shouldEqual None
     }
   }
@@ -122,28 +122,28 @@ class SchemaMapperTest extends AnyFeatureSpec with Matchers {
     Scenario("can generate mapper with exact fields matched by index") {
       val mapper = SchemaMapper(externalSchema, internalColumns)
 
-      mapper.tableColumn("externalId").get.name shouldEqual "id"
-      mapper.tableColumn("externalRic").get.name shouldEqual "ric"
-      mapper.tableColumn("assetClass").get.name shouldEqual "assetClass"
-      mapper.tableColumn("price").get.name shouldEqual "price"
+      mapper.getMappedColumn("externalId").get.name shouldEqual "id"
+      mapper.getMappedColumn("externalRic").get.name shouldEqual "ric"
+      mapper.getMappedColumn("assetClass").get.name shouldEqual "assetClass"
+      mapper.getMappedColumn("price").get.name shouldEqual "price"
     }
 
     Scenario("can generate mapper when an external field has no matched column") {
       val mapper = SchemaMapper(externalSchema, internalColumns.slice(0, 3))
 
-      mapper.tableColumn("externalId") shouldBe empty
-      mapper.tableColumn("externalRic").get.name shouldEqual "ric"
-      mapper.tableColumn("assetClass").get.name shouldEqual "assetClass"
-      mapper.tableColumn("price").get.name shouldEqual "price"
+      mapper.getMappedColumn("externalId") shouldBe empty
+      mapper.getMappedColumn("externalRic").get.name shouldEqual "ric"
+      mapper.getMappedColumn("assetClass").get.name shouldEqual "assetClass"
+      mapper.getMappedColumn("price").get.name shouldEqual "price"
     }
 
     Scenario("can generate mapper when a column has no matched external field") {
       val mapper = SchemaMapper(TestEntitySchema(externalFields.slice(0, 3)), internalColumns)
 
-      mapper.tableColumn("externalId").get.name shouldEqual "id"
-      mapper.tableColumn("externalRic") shouldBe empty
-      mapper.tableColumn("assetClass").get.name shouldBe  "assetClass"
-      mapper.tableColumn("price").get.name shouldEqual "price"
+      mapper.getMappedColumn("externalId").get.name shouldEqual "id"
+      mapper.getMappedColumn("externalRic") shouldBe empty
+      mapper.getMappedColumn("assetClass").get.name shouldBe  "assetClass"
+      mapper.getMappedColumn("price").get.name shouldEqual "price"
     }
   }
 }