Skip to content

Commit

Permalink
finos#1296 use parameterized SQL query builder
Browse files Browse the repository at this point in the history
- this would prevent SQL injection while adding better support
  for different types i.e. for instance before we had to manually
  convert values to strings (with or without quotes depending on
  the type) before using them in SQL query. Now, ignite would do
  that for us.
  • Loading branch information
junaidzm13 committed May 1, 2024
1 parent d618f9c commit 978bd83
Show file tree
Hide file tree
Showing 13 changed files with 264 additions and 136 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import org.apache.ignite.cluster.ClusterState
import org.apache.ignite.{IgniteCache, Ignition}
import org.finos.vuu.core.module.simul.model.{ChildOrder, OrderStore, ParentOrder}
import org.finos.vuu.example.ignite.utils.getListToObjectConverter
import org.finos.vuu.feature.ignite.IgniteSqlQuery
import org.finos.vuu.feature.ignite.IgniteSqlQuery.QuerySeparator

import java.util
import scala.collection.mutable
Expand Down Expand Up @@ -87,27 +89,32 @@ class IgniteOrderStore(private val parentOrderCache: IgniteCache[Int, ParentOrde

}

def getCount(sqlFilterQueries: String): Long = {
def getCount(filterSql: IgniteSqlQuery): Long = {
//todo should this be COUNT_BIG?
val whereClause = if(sqlFilterQueries == null || sqlFilterQueries.isEmpty) "" else s" where $sqlFilterQueries"
val query = new SqlFieldsQuery(s"select COUNT(1) from ChildOrder$whereClause")
val cursor = childOrderCache.query(query)
val whereClause = if (filterSql.isEmpty) filterSql else filterSql.prependSql("WHERE", QuerySeparator.SPACE)
val query = IgniteSqlQuery("SELECT COUNT(1) FROM ChildOrder").appendQuery(whereClause, QuerySeparator.SPACE)

val countValue = cursor.getAll().get(0).get(0)
val cursor = childOrderCache.query(query.buildFieldsQuery())
val countValue = cursor.getAll.get(0).get(0)
val totalCount = countValue.asInstanceOf[Long]

logger.info(s"Ignite returned total count of $totalCount for ChildOrder with filter $sqlFilterQueries")
logger.info(s"Ignite returned total count of `$totalCount` for ChildOrder with filter `$filterSql`")
totalCount
}

def findChildOrder(sqlFilterQueries: String, sqlSortQueries: String, rowCount: Int, startIndex: Long): Iterator[ChildOrder] = {
val whereClause = if(sqlFilterQueries == null || sqlFilterQueries.isEmpty) "" else s" where $sqlFilterQueries"
val orderByClause = if(sqlSortQueries == null || sqlSortQueries.isEmpty) " order by id" else s" order by $sqlSortQueries"
val query = new SqlFieldsQuery(s"select * from ChildOrder$whereClause$orderByClause limit ? offset ?")
query.setArgs(rowCount, startIndex)
def findChildOrder(filterSql: IgniteSqlQuery, sortSql: IgniteSqlQuery, rowCount: Int, startIndex: Long): Iterator[ChildOrder] = {
val whereClause = if (filterSql.isEmpty) filterSql else filterSql.prependSql("WHERE", QuerySeparator.SPACE)
val orderByClause = if (sortSql.isEmpty) IgniteSqlQuery("ORDER BY id") else sortSql.prependSql("ORDER BY", QuerySeparator.SPACE)
val limitAndOffsetClause = IgniteSqlQuery("limit ? offset ?", List(rowCount, startIndex))

val query = IgniteSqlQuery("SELECT * FROM ChildOrder")
.appendQuery(whereClause, QuerySeparator.SPACE)
.appendQuery(orderByClause, QuerySeparator.SPACE)
.appendQuery(limitAndOffsetClause, QuerySeparator.SPACE)

val results = childOrderCache.query(query).asScala.iterator.map(i => toChildOrder(i.asScala.toList))
logger.info(s"Loaded Ignite ChildOrder for $rowCount rows, from index : $startIndex where $whereClause order by $sqlSortQueries")
val results = childOrderCache.query(query.buildFieldsQuery()).asScala.iterator.map(i => toChildOrder(i.asScala.toList))
logger.info(s"Loaded Ignite ChildOrder for $rowCount rows, from index : $startIndex with " +
s"WHERE CLAUSE: `$whereClause` | ORDER BY CLAUSE: `$orderByClause`")

results
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import org.finos.vuu.example.ignite.module.IgniteOrderDataModule
import org.finos.vuu.example.ignite.provider.IgniteOrderDataProvider.columnNameByExternalField
import org.finos.vuu.example.ignite.query.IndexCalculator
import org.finos.vuu.example.ignite.schema.ChildOrderSchema
import org.finos.vuu.feature.ignite.IgniteSqlQuery
import org.finos.vuu.plugin.virtualized.table.{VirtualizedRange, VirtualizedSessionTable, VirtualizedViewPortKeys}
import org.finos.vuu.provider.VirtualizedProvider
import org.finos.vuu.util.schema.SchemaMapperBuilder
Expand Down Expand Up @@ -52,8 +53,7 @@ class IgniteOrderDataProvider(final val igniteStore: IgniteOrderStore)
viewPort.setKeys(new VirtualizedViewPortKeys(internalTable.primaryKeys))
}

private def getTotalSize(filter: String): Long =
igniteStore.getCount(filter)
private def getTotalSize(filterSql: IgniteSqlQuery): Long = igniteStore.getCount(filterSql)

private def tableUpdater(table: VirtualizedSessionTable): (Int, Map[String, Any]) => Unit = {
val keyField = table.tableDef.keyField
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package org.finos.vuu.example.ignite.provider
import org.finos.vuu.core.module.simul.model.ChildOrder
import org.finos.vuu.core.sort.ModelType.SortSpecInternal
import org.finos.vuu.example.ignite.IgniteOrderStore
import org.finos.vuu.feature.ignite.FilterAndSortSpecToSql
import org.finos.vuu.feature.ignite.{FilterAndSortSpecToSql, IgniteSqlQuery}
import org.finos.vuu.net.FilterSpec
import org.finos.vuu.util.schema.SchemaMapper

Expand All @@ -12,7 +12,7 @@ class IgniteOrderDataQuery private (private val igniteOrderStore: IgniteOrderSto

private val filterAndSortSpecToSql = FilterAndSortSpecToSql(schemaMapper)

def getFilterSql(filterSpec: FilterSpec): String =
def getFilterSql(filterSpec: FilterSpec): IgniteSqlQuery =
filterAndSortSpecToSql.filterToSql(filterSpec)

def fetch(filterSpec: FilterSpec, sortSpec: SortSpecInternal, startIndex: Long, rowCount: Int): Iterator[ChildOrder] = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package org.finos.vuu.example.ignite
import org.apache.ignite.cache.query.IndexQueryCriteriaBuilder
import org.apache.ignite.{Ignite, IgniteCache}
import org.finos.vuu.core.module.simul.model.{ChildOrder, ParentOrder}
import org.finos.vuu.feature.ignite.IgniteSqlQuery
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import org.scalatest.funsuite.AnyFunSuiteLike
import org.scalatest.matchers.should.Matchers
Expand All @@ -13,9 +14,6 @@ class IgniteOrderStoreTest extends AnyFunSuiteLike with BeforeAndAfterAll with B
private var childOrderCache: IgniteCache[Int, ChildOrder] = _
private var orderStore: IgniteOrderStore = _

private val emptySortQueries: String = ""
private val emptyFilterQueries: String = ""

override def beforeAll(): Unit = {
ignite = TestUtils.setupIgnite(testName = this.toString)
parentOrderCache = ignite.getOrCreateCache("parentOrderCache")
Expand Down Expand Up @@ -92,8 +90,8 @@ class IgniteOrderStoreTest extends AnyFunSuiteLike with BeforeAndAfterAll with B
parentOrder2 = GivenParentHasChildOrder(parentOrder2, 5)
parentOrder2 = GivenParentHasChildOrder(parentOrder2, 6)

val filterQueries = "parentId = 2"
val childOrder = orderStore.findChildOrder(filterQueries, emptySortQueries, 2, 1).toList
val filterQuery = IgniteSqlQuery("parentId = ?", List(2))
val childOrder = orderStore.findChildOrder(filterQuery, IgniteSqlQuery.empty, 2, 1).toList

assert(childOrder != null)
assert(childOrder.size == 2)
Expand All @@ -113,8 +111,8 @@ class IgniteOrderStoreTest extends AnyFunSuiteLike with BeforeAndAfterAll with B
parentOrder2 = GivenParentHasChildOrder(parentOrder2, 5, ric = "VOD.L")
parentOrder2 = GivenParentHasChildOrder(parentOrder2, 6, ric = "VOD.L")

val filterQueries = "ric = \'VOD.L\'"
val childOrder = orderStore.findChildOrder(filterQueries, emptySortQueries, 100, 0).toList
val filterQuery = IgniteSqlQuery("ric = ?", List("VOD.L"))
val childOrder = orderStore.findChildOrder(filterQuery, IgniteSqlQuery.empty, 100, 0).toList

assert(childOrder != null)
assert(childOrder.size == 6)
Expand All @@ -128,7 +126,7 @@ class IgniteOrderStoreTest extends AnyFunSuiteLike with BeforeAndAfterAll with B
parentOrder2 = GivenParentHasChildOrder(parentOrder2, 4)
parentOrder2 = GivenParentHasChildOrder(parentOrder2, 5)

val childOrder = orderStore.findChildOrder(emptyFilterQueries, emptySortQueries, 100, 0).toList
val childOrder = orderStore.findChildOrder(IgniteSqlQuery.empty, IgniteSqlQuery.empty, 100, 0).toList

assert(childOrder != null)
assert(childOrder.size == 3)
Expand All @@ -142,8 +140,8 @@ class IgniteOrderStoreTest extends AnyFunSuiteLike with BeforeAndAfterAll with B
var parentOrder2: ParentOrder = GivenParentOrder(2)
parentOrder2 = GivenParentHasChildOrder(parentOrder2, 2)

val sortByValues = "id ASC"
val childOrder = orderStore.findChildOrder(emptyFilterQueries, sortByValues, 100, 0).toList
val sortByValues = IgniteSqlQuery("id ASC")
val childOrder = orderStore.findChildOrder(IgniteSqlQuery.empty, sortByValues, 100, 0).toList

assert(childOrder != null)
assert(childOrder.size == 3)
Expand All @@ -158,8 +156,8 @@ class IgniteOrderStoreTest extends AnyFunSuiteLike with BeforeAndAfterAll with B
var parentOrder2: ParentOrder = GivenParentOrder(2)
parentOrder2 = GivenParentHasChildOrder(parentOrder2, 2)

val sortByValues = "parentId DESC, id ASC"
val childOrder = orderStore.findChildOrder(emptyFilterQueries, sortByValues, 100, 0).toList
val sortByValues = IgniteSqlQuery("parentId DESC, id ASC")
val childOrder = orderStore.findChildOrder(IgniteSqlQuery.empty, sortByValues, 100, 0).toList

assert(childOrder != null)
assert(childOrder.size == 3)
Expand All @@ -178,9 +176,9 @@ class IgniteOrderStoreTest extends AnyFunSuiteLike with BeforeAndAfterAll with B
parentOrder2 = GivenParentHasChildOrder(parentOrder2, 5, ric = "VOD.L")
parentOrder2 = GivenParentHasChildOrder(parentOrder2, 6, ric = "BP.L")

val filterQueries = "ric = \'VOD.L\'"
val filterQuery = IgniteSqlQuery("ric = ?", List("VOD.L"))

val count = orderStore.getCount(filterQueries)
val count = orderStore.getCount(filterQuery)

assert(count == 3)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import org.finos.vuu.core.sort.SortDirection
import org.finos.vuu.core.table.{Column, Columns}
import org.finos.vuu.example.ignite.IgniteOrderStore
import org.finos.vuu.example.ignite.provider.IgniteOrderDataQueryTest.{entitySchema, internalColumns, internalColumnsByExternalFields}
import org.finos.vuu.feature.ignite.IgniteSqlQuery
import org.finos.vuu.net.FilterSpec
import org.finos.vuu.util.schema.{ExternalEntitySchema, ExternalEntitySchemaBuilder, SchemaMapper, SchemaMapperBuilder}
import org.scalamock.scalatest.MockFactory
Expand All @@ -26,7 +27,8 @@ class IgniteOrderDataQueryTest extends AnyFeatureSpec with Matchers with MockFac
val filterSpec = FilterSpec("id = 2 and name != \"ABC\"")
val sortSpec = Map("value" -> SortDirection.Ascending)

(igniteStore.findChildOrder _).expects("(key = 2 AND name != 'ABC')", "value ASC", *, *).once()
(igniteStore.findChildOrder _)
.expects(IgniteSqlQuery("(key = ? AND name != ?)", List(2, "ABC")), IgniteSqlQuery("value ASC"), *, *).once()

igniteDataQuery.fetch(filterSpec, sortSpec, 0, 0)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ import org.finos.vuu.net.FilterSpec
import org.finos.vuu.util.schema.SchemaMapper

trait FilterAndSortSpecToSql {
def filterToSql(filterSpec: FilterSpec): String
def sortToSql(sortSpec: SortSpecInternal): String
def filterToSql(filterSpec: FilterSpec): IgniteSqlQuery
def sortToSql(sortSpec: SortSpecInternal): IgniteSqlQuery
}

object FilterAndSortSpecToSql {
Expand All @@ -21,14 +21,16 @@ private class FilterAndSortSpecToSqlImpl(private val schemaMapper: SchemaMapper)
private val filterTreeVisitor = new IgniteSqlFilterTreeVisitor
private val igniteSqlSortBuilder = new IgniteSqlSortBuilder

override def filterToSql(filterSpec: FilterSpec): String = {
override def filterToSql(filterSpec: FilterSpec): IgniteSqlQuery = {
if (filterSpec.filter == null || filterSpec.filter.isEmpty) {
""
IgniteSqlQuery.empty
} else {
val clause = FilterSpecParser.parse[IgniteSqlFilterClause](filterSpec.filter, filterTreeVisitor)
clause.toSql(schemaMapper)
}
}

override def sortToSql(sortSpec: SortSpecInternal): String = igniteSqlSortBuilder.toSql(sortSpec, schemaMapper)
override def sortToSql(sortSpec: SortSpecInternal): IgniteSqlQuery = {
if (sortSpec == null) IgniteSqlQuery.empty else igniteSqlSortBuilder.toSql(sortSpec, schemaMapper)
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package org.finos.vuu.feature.ignite

import org.apache.ignite.cache.query.SqlFieldsQuery
import org.finos.vuu.feature.ignite.IgniteSqlQuery.QuerySeparator

object IgniteSqlQuery {
def apply(sqlTemplate: String): IgniteSqlQuery = new IgniteSqlQuery(sqlTemplate, List.empty)
def apply(): IgniteSqlQuery = IgniteSqlQuery("")
def empty: IgniteSqlQuery = IgniteSqlQuery()

sealed abstract class QuerySeparator(val value: String)
object QuerySeparator {
final case object AND extends QuerySeparator(value = " AND ")
final case object OR extends QuerySeparator(value = " OR ")
final case object SPACE extends QuerySeparator(value = " ")
final case object EMPTY extends QuerySeparator(value = "")
}
}

case class IgniteSqlQuery(sqlTemplate: String, args: List[Any]) {

def appendSql(sqlTemplate: String, sep: QuerySeparator = QuerySeparator.EMPTY): IgniteSqlQuery = {
val newTemplate = if (sqlTemplate.isEmpty) this.sqlTemplate else Array(this.sqlTemplate, sqlTemplate).mkString(sep.value)
this.copy(sqlTemplate = newTemplate)
}

def prependSql(sqlTemplate: String, sep: QuerySeparator = QuerySeparator.EMPTY): IgniteSqlQuery = {
val newTemplate = if (sqlTemplate.isEmpty) this.sqlTemplate else Array(sqlTemplate, this.sqlTemplate).mkString(sep.value)
this.copy(sqlTemplate = newTemplate)
}

def appendArgs(args: List[Any]): IgniteSqlQuery = {
this.copy(args = this.args ++ args)
}

def appendQuery(query: IgniteSqlQuery, sep: QuerySeparator = QuerySeparator.EMPTY): IgniteSqlQuery = {
this.appendSql(query.sqlTemplate, sep).appendArgs(query.args)
}

def isEmpty: Boolean = this.sqlTemplate.isEmpty && this.args.isEmpty

def buildFieldsQuery(): SqlFieldsQuery = new SqlFieldsQuery(sqlTemplate).setArgs(args.toArray: _*)
}
Loading

0 comments on commit 978bd83

Please sign in to comment.