Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge pull request #24 from holdenk/master

Update ES score generation & add more logging
  • Loading branch information...
commit 2899a277f5cf47792242c6b3f7b946899b753460 2 parents 94d2fbd + 3812cb6
Adam Alix adamalix authored
2  build.sbt
View
@@ -1,6 +1,6 @@
name := "slashem"
-version := "0.9.8"
+version := "0.9.10"
organization := "com.foursquare"
28 src/main/scala/com/foursquare/slashem/Ast.scala
View
@@ -223,8 +223,8 @@ object Ast {
def boost(): String = {
fieldName
}
- def elasticBoost(): String = {
- "(doc['" + fieldName + "'].value)"
+ def elasticBoost(): Pair[List[String],String] = {
+ Pair(Nil,"(doc['" + fieldName + "'].value)")
}
}
@@ -240,10 +240,10 @@ object Ast {
case x: Double => fieldName + "^" + x.toString
}
}
- def elasticBoost(): String = {
+ def elasticBoost(): Pair[List[String],String] = {
weight match {
- case 1.0 => "(doc['" + fieldName + "'].value)"
- case _ => "(doc['" + fieldName + "'].value *" + weight.toString + ")"
+ case 1.0 => Pair(Nil,"(doc['" + fieldName + "'].value)")
+ case _ => Pair(Nil,"(doc['" + fieldName + "'].value *" + weight.toString + ")")
}
}
}
@@ -311,8 +311,10 @@ object Ast {
def boost(): String
/**
* Elastic Search field boost function
+ * The first param is a list of of params and
+ * the second is a string to which scala format could be applied
*/
- def elasticBoost(): String
+ def elasticBoost(): Pair[List[String],String]
}
/**
@@ -325,14 +327,14 @@ object Ast {
def boost(): String = {
distType match {
case "square" => "sqedist(%s,%s,%s)".format(lat,lng,name)
- case _ => "dist(2,%s,%s,%s)".format(lat,lng,name)
+ case _ => "dist(2,%s,%s,%s".format(lat,lng,name)
}
}
/** @inheritdoc */
- def elasticBoost(): String = {
- val distanceInKm = "doc['%s'].distanceInKm(%s,%s)".format(name,lat,lng)
+ def elasticBoost(): Pair[List[String],String] = {
+ val distanceInKm = Pair(List(lat,lng).map(_.toString),"doc['"+name+"'].distanceInKm(%s,%s)")
distType match {
- case "square" => "pow(%s,2.0)".format(distanceInKm)
+ case "square" => Pair(distanceInKm._1,"pow(%s,2.0)".format(distanceInKm._2))
case _ => distanceInKm
}
}
@@ -342,7 +344,11 @@ object Ast {
/** @inheritdoc */
def boost: String = "recip(%s,%d,%d,%d)".format(query.boost, x, y, z)
/** @inheritdoc */
- def elasticBoost(): String = "%d.0*pow(((%d.0*(%s))+%d.0),-1.0)".format(y, x, query.elasticBoost(), z)
+ def elasticBoost(): Pair[List[String],String] = {
+ val subquery= query.elasticBoost()
+ Pair(subquery._1,
+ "%d.0*pow(((%d.0*(%s))+%d.0),-1.0)".format(y, x, subquery._2, z))
+ }
}
/**
42 src/main/scala/com/foursquare/slashem/Schema.scala
View
@@ -390,12 +390,16 @@ trait SolrQueryLogger {
//Log success
def success(name: String): Unit = {
}
+ //Log the number of results
+ def resultCount(name: String, count: Int): Unit = {
+ }
}
/** The default logger, does nothing. */
object NoopQueryLogger extends SolrQueryLogger {
override def log(name: String, msg: String, time: Long) = Unit
override def debug(msg: String) = println(msg)
+ override def resultCount(name: String, count:Int) = println("Got back "+count+" results while querying "+name)
}
//If you want any of the geo queries you will have to implement this
@@ -483,9 +487,19 @@ trait ElasticSchema[M <: Record[M]] extends SlashemSchema[M] {
case Some(Pair(Field(fieldName),"asc")) => baseRequest.addSort(fieldName,SortOrder.ASC)
case Some(Pair(Field(fieldName),"desc")) => baseRequest.addSort(fieldName,SortOrder.DESC)
//Handle sorting by scripts in general
- case Some(Pair(sort,"asc")) => baseRequest.addSort(new ScriptSortBuilder(sort.elasticBoost(),"number").order(SortOrder.ASC))
- case Some(Pair(sort,"desc")) => baseRequest.addSort(new ScriptSortBuilder(sort.elasticBoost(),"number").order(SortOrder.DESC))
-
+ case Some(Pair(sort,dir)) => {
+ val (params,scriptSrc) = sort.elasticBoost()
+ val paramNames = (1 to params.length).map("p"+_)
+ val script = scriptSrc.format(paramNames:_*)
+ val keyedParams = paramNames zip params
+ val sortOrder = dir match {
+ case "asc" => SortOrder.ASC
+ case "desc" => SortOrder.DESC
+ }
+ val sortBuilder = new ScriptSortBuilder(script,"number").order(sortOrder)
+ keyedParams.foreach(p => {sortBuilder.param(p._1,p._2)})
+ baseRequest.addSort(sortBuilder)
+ }
case _ => baseRequest
}
@@ -507,9 +521,11 @@ trait ElasticSchema[M <: Record[M]] extends SlashemSchema[M] {
response
}
+ val queryText = query.toString()
+
timeFuture(searchResultsFuture).map( {
case (queryTime, result) => {
- meta.logger.log("e" + meta.indexName + ".query",query.toString(), queryTime)
+ meta.logger.log("e" + meta.indexName + ".query",queryText, queryTime)
result
}}).map({
response =>
@@ -522,6 +538,11 @@ trait ElasticSchema[M <: Record[M]] extends SlashemSchema[M] {
results
})
+ .onSuccess((v: SearchResults[M,Y]) => {
+ meta.logger.success("e"+meta.indexName)
+ meta.logger.resultCount("e"+meta.indexName,v.response.numFound)})
+ .onFailure(e => meta.logger.failure("e"+meta.indexName, queryText, e))
+
}
def constructSearchResults[Y](creator: Option[Response.RawDoc => Y],
start: Int,
@@ -596,7 +617,14 @@ trait ElasticSchema[M <: Record[M]] extends SlashemSchema[M] {
}
def boostFields(query: ElasticQueryBuilder, boostFields: List[ScoreBoost]): ElasticQueryBuilder = {
val boostedQuery = new CustomScoreQueryBuilder(query)
- val scoreScript = "_score * (1 +"+(boostFields.map(_.elasticBoost).mkString(" + ") + " )")
+ val boostedQuerys = boostFields.map(_.elasticBoost)
+ val params = boostedQuerys.flatMap(_._1)
+ val scriptSrc = boostedQuerys.map(_._2).mkString(" + ")
+ val paramNames = (1 to params.length).map("p"+_)
+ val script = scriptSrc.format(paramNames:_*)
+ val keyedParams = paramNames zip params
+ keyedParams.foreach(p => {boostedQuery.param(p._1,p._2)})
+ val scoreScript = "_score * (1 +"+ script + " )"
boostedQuery.script(scoreScript)
}
def combineFilters(filters: List[ElasticFilterBuilder]): ElasticFilterBuilder = {
@@ -723,7 +751,9 @@ trait SolrSchema[M <: Record[M]] extends SlashemSchema[M] {
min,
queryText)
})
- .onSuccess(_ => meta.logger.success(meta.solrName))
+ .onSuccess((v: SearchResults[M,Y]) => {
+ meta.logger.success(meta.solrName)
+ meta.logger.resultCount(meta.solrName,v.response.numFound)})
.onFailure(e => meta.logger.failure(meta.solrName, queryText, e))
}
Please sign in to comment.
Something went wrong with that request. Please try again.