Skip to content

Commit

Permalink
finos#1167 add small enhacements in ExternalEntitySchema
Browse files Browse the repository at this point in the history 
- changes mainly around better naming and types, resulting
  in easier to use API.
  • Loading branch information
@junaidzm13
junaidzm13 committed Mar 18, 2024
1 parent 99e35ee commit cc30a88
Show file tree
Hide file tree
Showing 9 changed files with 68 additions and 64 deletions.
8 changes: 4 additions & 4 deletions ...he-ignite/src/main/scala/org/finos/vuu/example/ignite/schema/BaseIgniteEntityObject.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@ object BaseIgniteEntityObject {

def buildQueryEntity(schema: ExternalEntitySchema, keyClass: Class[_], valueClass: Class[_]): QueryEntity = {
val fields = new java.util.LinkedHashMap[String, String](
mutable.LinkedHashMap.empty.addAll(schema.schemaFields.map(f => (f.name, f.dType.getName))).asJava
mutable.LinkedHashMap.empty.addAll(schema.fields.map(f => (f.name, f.dataType.getName))).asJava
)

val queryIndex = schema.index.map({ case (indexName, fields) =>
new QueryIndex(fields.asJavaCollection, QueryIndexType.SORTED).setName(indexName)
})
val queryIndex = schema.indexes.map(index =>
new QueryIndex(index.fields.asJavaCollection, QueryIndexType.SORTED).setName(index.name)
)

new QueryEntity(keyClass, valueClass).setFields(fields).setIndexes(queryIndex.asJava)
}
Expand Down
14 changes: 7 additions & 7 deletions ...ite-plugin/src/main/scala/org/finos/vuu/feature/ignite/filter/IgniteSqlFilterClause.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ case class AndIgniteSqlFilterClause(clauses:List[IgniteSqlFilterClause]) extends
case class EqIgniteSqlFilterClause(columnName: String, value: String) extends IgniteSqlFilterClause {
override def toSql(schemaMapper: SchemaMapper): String =
schemaMapper.externalSchemaField(columnName) match {
case Some(f) => f.dType match {
case Some(f) => f.dataType match {
case CharDataType | StringDataType => eqSql(f.name, quotedString(value))
case _ => eqSql(f.name, value)
}
Expand All @@ -45,7 +45,7 @@ case class EqIgniteSqlFilterClause(columnName: String, value: String) extends Ig
case class NeqIgniteSqlFilterClause(columnName: String, value: String) extends IgniteSqlFilterClause {
override def toSql(schemaMapper: SchemaMapper): String =
schemaMapper.externalSchemaField(columnName) match {
case Some(field) => field.dType match {
case Some(field) => field.dataType match {
case CharDataType | StringDataType => neqSql(field.name, quotedString(value))
case _ => neqSql(field.name, value)
}
Expand All @@ -70,9 +70,9 @@ case class RangeIgniteSqlFilterClause(op: RangeOp)(columnName: String, value: Do
case class StartsIgniteSqlFilterClause(columnName: String, value: String) extends IgniteSqlFilterClause with StrictLogging {
override def toSql(schemaMapper: SchemaMapper): String = {
schemaMapper.externalSchemaField(columnName) match {
case Some(f) => f.dType match {
case Some(f) => f.dataType match {
case StringDataType => s"${f.name} LIKE '$value%'"
case _ => logErrorAndReturnEmptySql(s"`Starts` clause unsupported for non string column: `${f.name}` (${f.dType})")
case _ => logErrorAndReturnEmptySql(s"`Starts` clause unsupported for non string column: `${f.name}` (${f.dataType})")
}
case None => logMappingErrorAndReturnEmptySql(columnName)
}
Expand All @@ -82,9 +82,9 @@ case class StartsIgniteSqlFilterClause(columnName: String, value: String) extend
case class EndsIgniteSqlFilterClause(columnName: String, value: String) extends IgniteSqlFilterClause with StrictLogging {
override def toSql(schemaMapper: SchemaMapper): String =
schemaMapper.externalSchemaField(columnName) match {
case Some(f) => f.dType match {
case Some(f) => f.dataType match {
case StringDataType => s"${f.name} LIKE '%$value'"
case _ => logErrorAndReturnEmptySql(s"`Ends` clause unsupported for non string column: `${f.name}` (${f.dType})")
case _ => logErrorAndReturnEmptySql(s"`Ends` clause unsupported for non string column: `${f.name}` (${f.dataType})")
}
case None => logMappingErrorAndReturnEmptySql(columnName)
}
Expand All @@ -93,7 +93,7 @@ case class EndsIgniteSqlFilterClause(columnName: String, value: String) extends
case class InIgniteSqlFilterClause(columnName: String, values: List[String]) extends IgniteSqlFilterClause with StrictLogging {
override def toSql(schemaMapper: SchemaMapper): String =
schemaMapper.externalSchemaField(columnName) match {
case Some(f) => f.dType match {
case Some(f) => f.dataType match {
case CharDataType | StringDataType => inQuery(f.name, values.map(quotedString(_)))
case _ => inQuery(f.name, values)
}
Expand Down
2 changes: 1 addition & 1 deletion ...te-plugin/src/test/scala/org/finos/vuu/feature/ignite/filter/IgniteSqlFilteringTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -475,7 +475,7 @@ class IgniteSqlFilteringTest extends IgniteTestsBase {
).zipWithIndex.map({ case ((name, t), i) => SimpleColumn(name, i, t) })

private class TestEntitySchema extends ExternalEntitySchema {
override val schemaFields: List[SchemaField] = List(
override val fields: List[SchemaField] = List(
SchemaField("id", classOf[Int], 0),
SchemaField("parentId", classOf[String], 1),
SchemaField("ric", classOf[String], 2),
Expand Down
56 changes: 28 additions & 28 deletions vuu/src/main/scala/org/finos/vuu/util/schema/ExternalEntitySchema.scala
View file
Original file line number Diff line number Diff line change
@@ -1,30 +1,23 @@
package org.finos.vuu.util.schema

import org.finos.vuu.util.schema.EntitySchema.{ColumnName, Index, IndexName}
import org.finos.vuu.util.schema.EntitySchema.{FieldName, IndexName}
import org.finos.vuu.util.schema.ExternalDataType.{ExternalDataType, fromString}
import org.finos.vuu.util.schema.ExternalEntitySchemaBuilder.InvalidIndexException
import org.finos.vuu.util.schema.ExternalEntitySchemaBuilder.{InvalidIndexException, toSchemaFields}

import scala.collection.mutable
import scala.collection.mutable.ListBuffer
import scala.reflect.runtime.universe.{MethodSymbol, TypeTag, typeOf}

trait ExternalEntitySchema {
val schemaFields: List[SchemaField]
val index: List[Index] = List.empty
val fields: List[SchemaField]
val indexes: List[SchemaIndex] = List.empty
}

case class DefaultExternalEntitySchema private (private val columnDef: mutable.LinkedHashMap[ColumnName, ExternalDataType],
override val index: List[Index]) extends ExternalEntitySchema {

override val schemaFields: List[SchemaField] = columnDef.zipWithIndex.map(
{case ((name, dType), i) => SchemaField(name, dType, i)}
).toList
}
private case class DefaultExternalEntitySchema private (override val fields: List[SchemaField],
override val indexes: List[SchemaIndex]) extends ExternalEntitySchema

object EntitySchema {
type ColumnName = String
type FieldName = String
type IndexName = String
type Index = (IndexName, List[ColumnName])
}

object ExternalDataType extends Enumeration {
Expand All @@ -33,56 +26,63 @@ object ExternalDataType extends Enumeration {
val String : ExternalDataType = classOf[String]
val Double : ExternalDataType = classOf[Double]
val Long : ExternalDataType = classOf[Long]
val Char : ExternalDataType = classOf[Char]

def fromString(s: String): ExternalDataType = {
s.trim.toLowerCase match {
case "string" => ExternalDataType.String
case "double" => ExternalDataType.Double
case "int" => ExternalDataType.Int
case "long" => ExternalDataType.Long
case "char" => ExternalDataType.Char
case _ => throw new RuntimeException(s"Unsupported type passed: $s")
}
}
}

object ExternalEntitySchemaBuilder {
def apply(fieldDef: mutable.LinkedHashMap[ColumnName, ExternalDataType] = mutable.LinkedHashMap.empty,
index: ListBuffer[Index] = ListBuffer.empty): ExternalEntitySchemaBuilder =
new ExternalEntitySchemaBuilder(fieldDef, index)
def apply(): ExternalEntitySchemaBuilder = ExternalEntitySchemaBuilder(ListBuffer.empty, ListBuffer.empty)

def apply(fields: ListBuffer[(FieldName, ExternalDataType)],
indexes: ListBuffer[SchemaIndex]): ExternalEntitySchemaBuilder =
new ExternalEntitySchemaBuilder(fields, indexes)

private def toSchemaFields(fields: ListBuffer[(FieldName, ExternalDataType)]) =
fields.zipWithIndex.map({ case ((name, dType), i) => SchemaField(name, dType, i) }).toList

final class InvalidIndexException(error: String) extends RuntimeException(error)
}

case class ExternalEntitySchemaBuilder(private val fieldDef: mutable.LinkedHashMap[ColumnName, ExternalDataType],
private val index: ListBuffer[Index]) {
case class ExternalEntitySchemaBuilder private (private val fields: ListBuffer[(FieldName, ExternalDataType)],
private val indexes: ListBuffer[SchemaIndex]) {

def withColumn(columnName: ColumnName, dataType: ExternalDataType): ExternalEntitySchemaBuilder =
new ExternalEntitySchemaBuilder(fieldDef.addOne(columnName -> dataType), index)
def withField(fieldName: FieldName, dataType: ExternalDataType): ExternalEntitySchemaBuilder =
this.copy(fields = fields.addOne(fieldName -> dataType))

def withIndex(indexName: IndexName, fields: List[ColumnName]): ExternalEntitySchemaBuilder =
new ExternalEntitySchemaBuilder(fieldDef, index :+ (indexName, fields))
def withIndex(indexName: IndexName, fields: List[FieldName]): ExternalEntitySchemaBuilder =
this.copy(indexes = indexes :+ SchemaIndex(indexName, fields))

def withCaseClass[T: TypeTag]: ExternalEntitySchemaBuilder = {
val namesToTypes = typeOf[T].members.sorted.collect {
case m: MethodSymbol if m.isCaseAccessor =>
m.name.toString -> fromString(m.returnType.toString)
}
new ExternalEntitySchemaBuilder(fieldDef.addAll(namesToTypes), index)
this.copy(fields = fields.addAll(namesToTypes))
}

def build(): ExternalEntitySchema = {
val validationError = validateSchema
if (validationError.nonEmpty) throw new InvalidIndexException(validationError.get)

DefaultExternalEntitySchema(fieldDef, index.toList)
DefaultExternalEntitySchema(toSchemaFields(fields), indexes.toList)
}

private type ValidationError = Option[String]
private def validateSchema = indexAppliedOnAbsentField()
private def indexAppliedOnAbsentField(): ValidationError = {
val error = index
.flatMap({ case (indexName, fields) => fields.map((indexName, _)) })
.filter({ case (_, f) => !fieldDef.contains(f) })
val error = indexes
.flatMap(idx => idx.fields.map((idx.name, _)))
.filter({ case (_, f) => !fields.exists(field => field._1 == f) })
.zipWithIndex
.map({ case ((indexName, f), i) => s"${i + 1}) Field `$f` in index `$indexName` not found in schema." })
.mkString(" ")
Expand Down
2 changes: 1 addition & 1 deletion vuu/src/main/scala/org/finos/vuu/util/schema/SchemaField.scala
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
package org.finos.vuu.util.schema

case class SchemaField(name: String, dType: Class[_], index: Int)
case class SchemaField(name: String, dataType: Class[_], index: Int)
3 changes: 3 additions & 0 deletions vuu/src/main/scala/org/finos/vuu/util/schema/SchemaIndex.scala
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
package org.finos.vuu.util.schema

case class SchemaIndex(name: String, fields: List[String])
4 changes: 2 additions & 2 deletions vuu/src/main/scala/org/finos/vuu/util/schema/SchemaMapper.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ object SchemaMapper {
private def externalFieldsInMapConformsToExternalSchema(externalSchema: ExternalEntitySchema,
externalFields: Iterable[String]): ValidationError = {
externalFields
.find(field => externalSchema.schemaFields.forall(_.name != field))
.find(field => externalSchema.fields.forall(_.name != field))
.map(f => s"Field `$f` not found in external schema")
}

Expand Down Expand Up @@ -69,7 +69,7 @@ private class SchemaMapperImpl(private val externalSchema: ExternalEntitySchema,
override def toInternalRowMap(dto: Product): Map[String, Any] = toInternalRowMap(dto.productIterator.toList)

private def getExternalSchemaFieldsByColumnName =
externalSchema.schemaFields.flatMap(f =>
externalSchema.fields.flatMap(f =>
Option.when(columnNameByExternalField.contains(f.name))(columnNameByExternalField(f.name), f)
).toMap

Expand Down
41 changes: 21 additions & 20 deletions vuu/src/test/scala/org/finos/vuu/util/schema/ExternalEntitySchemaTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,26 +8,26 @@ import org.scalatest.prop.TableDrivenPropertyChecks._
class ExternalEntitySchemaTest extends AnyFeatureSpec with Matchers {
Feature("ExternalEntitySchemaBuilder") {
Scenario("Builder can correctly pass index to the schema when index applied to existent fields") {
val index1 = ("NAME_IDX", List("name"))
val index2 = ("SIZE_IDX", List("size"))
val index1 = SchemaIndex("NAME_IDX", List("name"))
val index2 = SchemaIndex("SIZE_IDX", List("size"))

val schema = ExternalEntitySchemaBuilder()
.withColumn("name", classOf[String])
.withColumn("size", classOf[Int])
.withIndex(index1._1, index1._2)
.withIndex(index2._1, index2._2)
.withField("name", classOf[String])
.withField("size", classOf[Int])
.withIndex(index1.name, index1.fields)
.withIndex(index2.name, index2.fields)
.build()

schema.index shouldEqual List(index1, index2)
schema.indexes shouldEqual List(index1, index2)
}

Scenario("Builder throws when user tries to build a schema with index applied to a non-existent field") {
val badIndex = ("BAD_IDX", List("missing-field"))
val badIndex = SchemaIndex("BAD_IDX", List("missing-field"))

val exception = intercept[InvalidIndexException](
ExternalEntitySchemaBuilder()
.withColumn("present-field", classOf[String])
.withIndex(badIndex._1, badIndex._2)
.withField("present-field", classOf[String])
.withIndex(badIndex.name, badIndex.fields)
.build()
)

Expand All @@ -36,15 +36,15 @@ class ExternalEntitySchemaTest extends AnyFeatureSpec with Matchers {
}

Scenario("Builder throws when user tries to build a schema with multiple indexes applied to multiple non-existent fields") {
val badIndex1 = ("BAD_IDX", List("missing-field-1", "missing-field-2"))
val badIndex2 = ("BAD_IDX2", List("missing-field-3"))
val badIndex1 = SchemaIndex("BAD_IDX", List("missing-field-1", "missing-field-2"))
val badIndex2 = SchemaIndex("BAD_IDX2", List("missing-field-3"))

val exception = intercept[InvalidIndexException](
ExternalEntitySchemaBuilder()
.withColumn("present-field-1", classOf[String])
.withColumn("present-field-2", classOf[String])
.withIndex(badIndex1._1, badIndex1._2)
.withIndex(badIndex2._1, badIndex2._2)
.withField("present-field-1", classOf[String])
.withField("present-field-2", classOf[String])
.withIndex(badIndex1.name, badIndex1.fields)
.withIndex(badIndex2.name, badIndex2.fields)
.build()
)

Expand All @@ -56,11 +56,11 @@ class ExternalEntitySchemaTest extends AnyFeatureSpec with Matchers {

Scenario("Can build schema by passing each field") {
val schema = ExternalEntitySchemaBuilder()
.withColumn("name", classOf[String])
.withColumn("size", classOf[Int])
.withField("name", classOf[String])
.withField("size", classOf[Int])
.build()

schema.schemaFields shouldEqual List(
schema.fields shouldEqual List(
SchemaField("name", classOf[String], 0),
SchemaField("size", classOf[Int], 1)
)
Expand All @@ -69,7 +69,7 @@ class ExternalEntitySchemaTest extends AnyFeatureSpec with Matchers {
Scenario("Can build schema with a case class") {
val schema = ExternalEntitySchemaBuilder().withCaseClass[TestCaseClass].build()

schema.schemaFields shouldEqual List(
schema.fields shouldEqual List(
SchemaField("name", classOf[String], 0),
SchemaField("size", classOf[Int], 1),
SchemaField("value", classOf[Double], 2)
Expand All @@ -85,6 +85,7 @@ class ExternalEntitySchemaTest extends AnyFeatureSpec with Matchers {
("int", ExternalDataType.Int),
("long", ExternalDataType.Long),
("double", ExternalDataType.Double),
("char", ExternalDataType.Char),
))((str, expected) =>
Scenario(
s"can convert `$str` to correct ignite data type"
Expand Down
2 changes: 1 addition & 1 deletion vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ class SchemaMapperTest extends AnyFeatureSpec with Matchers {
}

private class TestEntitySchema extends ExternalEntitySchema {
override val schemaFields: List[SchemaField] = List(
override val fields: List[SchemaField] = List(
SchemaField("externalId", classOf[Int], 0),
SchemaField("externalRic", classOf[String], 1),
SchemaField("assetClass", classOf[String], 2),
Expand Down

0 comments on commit cc30a88

Please sign in to comment.