Permalink
Browse files

Merge pull request #44 from holdenk/master

Update slashem
  • Loading branch information...
adamalix committed Jun 26, 2012
2 parents 6d146a4 + 5e3124f commit 721bbae461540fb4b195296799b5212336d1e667
View
@@ -1,6 +1,6 @@
name := "slashem"
-version := "0.12.2"
+version := "0.13.0"
organization := "com.foursquare"
@@ -111,7 +111,8 @@ case class QueryBuilder[M <: Record[M], Ord, Lim, MM <: MinimumMatchType, Y, H <
@param f The boost query
*/
def boostQuery[F](f: M => Clause[F]): QueryBuilder[M, Ord, Lim, MM, Y, H, Q, MinFacetCount, FacetLimit, ST] = {
- this.copy(boostQueries=f(meta) :: boostQueries)
+ val newclause = f(meta)
+ this.copy(boostQueries=newclause :: boostQueries)
}
/** Helper method for case class extraction */
@@ -7,6 +7,7 @@ import com.foursquare.slashem.Ast._
import com.twitter.util.{Duration, ExecutorServiceFuturePool, Future, FuturePool, FutureTask, Promise}
import com.twitter.finagle.builder.ClientBuilder
import com.twitter.finagle.http.Http
+import com.twitter.finagle.stats.StatsReceiver
import com.twitter.finagle.Service
import java.lang.Integer
import java.net.InetSocketAddress
@@ -28,9 +29,12 @@ import org.elasticsearch.common.settings.ImmutableSettings
import org.elasticsearch.common.transport.InetSocketTransportAddress
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilders.filteredQuery
-import org.elasticsearch.index.query.{AndFilterBuilder, CustomScoreQueryBuilder,
+import org.elasticsearch.index.query.{AndFilterBuilder,
+ BoostingQueryBuilder,
+ CustomScoreQueryBuilder,
FilterBuilder => ElasticFilterBuilder,
- QueryBuilder => ElasticQueryBuilder}
+ QueryBuilder => ElasticQueryBuilder,
+ QueryBuilders => EQueryBuilders}
import org.elasticsearch.node.Node
import org.elasticsearch.node.NodeBuilder._
import org.elasticsearch.search.facet.AbstractFacetBuilder
@@ -280,6 +284,9 @@ trait SolrMeta[T <: Record[T]] extends SlashemMeta[T] {
}
}
+ /* Want to collect some finagle stats, provide a Stats receiver */
+ def receiver: Option[StatsReceiver] = None
+
// The name is used to determine which props to use as well as for logging
def solrName: String
@@ -295,18 +302,23 @@ trait SolrMeta[T <: Record[T]] extends SlashemMeta[T] {
case Some(cl) => cl
case _ => {
myClient = Some({
- ClientBuilder()
- .codec(Http())
- .hosts(servers.map(x => {
- val h = x.split(":")
- val s = h.head
- val p = h.last
- new InetSocketAddress(s, p.toInt)
- }))
- .hostConnectionLimit(hostConnectionLimit)
- .hostConnectionCoresize(hostConnectionCoresize)
- .retries(retries)
- .build()})
+ val cb = ClientBuilder()
+ .codec(Http())
+ .hosts(servers.map(x => {
+ val h = x.split(":")
+ val s = h.head
+ val p = h.last
+ new InetSocketAddress(s, p.toInt)
+ }))
+ .hostConnectionLimit(hostConnectionLimit)
+ .hostConnectionCoresize(hostConnectionCoresize)
+ .retries(retries)
+ .name(solrName)
+ (receiver match {
+ case Some(r) => cb.reportTo(r)
+ case _ => cb
+ }).build()
+ })
myClient.get
}
}
@@ -572,6 +584,7 @@ trait ElasticSchema[M <: Record[M]] extends SlashemSchema[M] {
.onFailure(e => meta.logger.failure("e"+meta.indexName, queryText, e))
}
+
def constructSearchResults[Y](creator: Option[Response.RawDoc => Y],
start: Int,
fallOff: Option[Double],
@@ -631,7 +644,12 @@ trait ElasticSchema[M <: Record[M]] extends SlashemSchema[M] {
}
case _ => scoreFields(fq, qb.boostFields)
}
- scoredQuery
+ //Apply query boosting
+ val boostedQuery = qb.boostQueries match {
+ case (x::xs) => boostQueries(scoredQuery, qb)
+ case _ => scoredQuery
+ }
+ boostedQuery
}
def termFacetQuery(facetFields: List[Ast.Field], facetLimit: Option[Int]): List[AbstractFacetBuilder] = {
@@ -648,6 +666,42 @@ trait ElasticSchema[M <: Record[M]] extends SlashemSchema[M] {
facetQueries
}
+ /**
+ * Applies positive and negative query boosts
+ */
+ def boostQueries(query: ElasticQueryBuilder, qb: QueryBuilder[_, _, _, _, _, _, _, _, _, _]): ElasticQueryBuilder = {
+ //Only bother making boost queries if we have negative boost queries otherwise we just append it
+ val boostQueries = qb.boostQueries
+ val negativeQueries = boostQueries.filter(q => q match {
+ case Clause(_,_,false) => true
+ case _ => false
+ })
+ if (negativeQueries.length > 0) {
+ val boostedQuery = new BoostingQueryBuilder()
+ boostedQuery.positive(query)
+ boostedQuery.negative(query)
+ boostQueries.map(q => q match {
+ case Clause(_,_,false) => boostedQuery.negative(q.elasticExtend(qb.queryFields,
+ qb.phraseBoostFields,
+ qb.minimumMatch))
+ case _ => boostedQuery.positive(q.elasticExtend(qb.queryFields,
+ qb.phraseBoostFields,
+ qb.minimumMatch))
+ })
+ boostedQuery.negativeBoost(0.1.toFloat)
+ boostedQuery
+ } else {
+ val orQuery = EQueryBuilders.boolQuery
+ orQuery.should(query)
+ boostQueries.map(q => orQuery.should(q.elasticExtend(qb.queryFields,
+ qb.phraseBoostFields,
+ qb.minimumMatch)
+ )
+ )
+ orQuery
+ }
+ }
+
/**
* Custom score the fields which have scoreboosts
*/
@@ -662,6 +716,7 @@ trait ElasticSchema[M <: Record[M]] extends SlashemSchema[M] {
scoreWithScript(query, scoreScript, namesAndParams, false)
}
+
/**
* Add the provided script and its params to the query and build a
* CustomScoreQuery with it.
@@ -937,6 +992,7 @@ class SlashemIntListField[T <: Record[T]](owner: T) extends IntListField[T](owne
case "" => Empty
case ar: Array[Int] => Full(ar.toList)
case ar: Array[Integer] => Full(ar.toList.map(x=>x.intValue))
+ case ar: ArrayList[_] => Full(set(ar.toArray.toList.map(x=>x.asInstanceOf[Integer].intValue)))
case s: String => Full(s.split(" ").map(x => x.toInt).toList)
case _ => Empty
}
@@ -1171,6 +1227,7 @@ class IntListField[T <: Record[T]](override val owner: T) extends Field[List[Int
case "" => Empty
case ar: Array[Int] => Full(set(ar.toList))
case ar: Array[Integer] => Full(set(ar.toList.map(x=>x.intValue)))
+ case ar: ArrayList[_] => Full(set(ar.toArray.toList.map(x=>x.asInstanceOf[Integer].intValue)))
case s: String => Full(set(s.split(" ").map(x => x.toInt).toList))
case _ => Empty
}
@@ -93,6 +93,30 @@ class ElasticQueryTest extends SpecsMatchers with ScalaCheckMatchers {
val r = fullQuery fetch()
}
+ @Test
+ def testBoostQuery {
+ val rLolerNyet = ESimplePanda where (_.name contains "loler") and (_.hobos contains "nyet") fetch()
+ val rBoostedNyet = ESimplePanda where (_.name contains "loler") boostQuery(_.hobos contains "nyet") fetch()
+ val rNoBoostedNyet = ESimplePanda where (_.name contains "loler") fetch()
+ Assert.assertEquals(1,rLolerNyet.response.results.length)
+ Assert.assertEquals(4,rBoostedNyet.response.results.length)
+ Assert.assertEquals(4,rNoBoostedNyet.response.results.length)
+ Assert.assertEquals(rBoostedNyet.response.results.apply(0).id.is,rLolerNyet.response.results.apply(0).id.is)
+ Assert.assertTrue(rBoostedNyet.response.results.apply(0).id.is != rNoBoostedNyet.response.results.apply(0).id.is)
+ }
+
+ @Test
+ def testNegativeBoostQuery {
+ val rLolerNyet = ESimplePanda where (_.name contains "loler") and (_.hobos.neqs("nyet")) fetch()
+ val rBoostedNyet = ESimplePanda where (_.name contains "loler") boostQuery(_.hobos.neqs("nyet")) fetch()
+ val rNoBoostedNyet = ESimplePanda where (_.name contains "loler") fetch()
+ Assert.assertEquals(3,rLolerNyet.response.results.length)
+ Assert.assertEquals(4,rBoostedNyet.response.results.length)
+ Assert.assertEquals(4,rNoBoostedNyet.response.results.length)
+ Assert.assertTrue(rBoostedNyet.response.results.apply(0).id.is != rNoBoostedNyet.response.results.apply(0).id.is)
+ }
+
+
@Test
def testEmptySearch {
@@ -247,7 +271,7 @@ class ElasticQueryTest extends SpecsMatchers with ScalaCheckMatchers {
val containsCount = rContains.response.results.length
Assert.assertTrue(containsCount > phraseCount)
}
- //@Test
+ @Test
def testFieldFaceting {
val r = ESimplePanda where (_.name contains "loler skates") facetField(_.foreign) fetch()
Assert.assertEquals(4,r.response.results.length)
@@ -329,6 +353,13 @@ class ElasticQueryTest extends SpecsMatchers with ScalaCheckMatchers {
Assert.assertEquals(response10.response.results.length, 1)
}
+ @Test
+ def testIntListFieldReturn {
+ val response1 = ESimplePanda where (_.favnums contains 2) fetch()
+ Assert.assertEquals(response1.response.results.length, 2)
+ Assert.assertEquals(response1.response.results.head.favnums.get, List(1,2,3,4,5))
+ }
+
@Test
def testListFieldIn {
val response1 = ESimplePanda where (_.favnums in List(2, 3, 4, 5)) fetch()
@@ -421,6 +452,9 @@ class ElasticQueryTest extends SpecsMatchers with ScalaCheckMatchers {
@Before
+ def setup() {
+ hoboPrepIndex()
+ }
def hoboPrepIndex() {
ESimplePanda.meta.node = ElasticNode.node
ESimpleGeoPanda.meta.node = ElasticNode.node
@@ -608,6 +642,9 @@ class ElasticQueryTest extends SpecsMatchers with ScalaCheckMatchers {
}
@After
+ def done() {
+ hoboDone()
+ }
def hoboDone() {
ESimplePanda.meta.node = ElasticNode.node
ESimpleGeoPanda.meta.node = ElasticNode.node
@@ -623,6 +660,30 @@ class ElasticQueryTest extends SpecsMatchers with ScalaCheckMatchers {
}
}
+//Optimize tests
+
+ //We test this because the optimizer screwed this up once
+ @Test
+ def testTermFiltersOptimize {
+ val q = ESimplePanda where (_.hugenums contains 1L) filter(_.termsfield in List("termhit", "randomterm"))
+ val res1 = q fetch()
+ val res2 = q optimize() fetch()
+ Assert.assertEquals(res1.response.results.length, res2.response.results.length)
+ }
+
+ @Test
+ def testTermFiltersMetallFilter {
+ val q = ESimpleGeoPanda where (_.name contains "ordertest") filter(_.metall any)
+ val q2 = ESimpleGeoPanda where (_.name contains "ordertest")
+ val res1 = q fetch()
+ val res2 = q optimize() fetch()
+ val res3 = q2 fetch()
+ Assert.assertEquals(res1.response.results.length, res2.response.results.length)
+ Assert.assertEquals(res1.response.results.length, res3.response.results.length)
+ Assert.assertEquals(q.optimize(),q2)
+ }
+
+
}
object ElasticQueryTest extends ElasticQueryTest {
@@ -29,27 +29,6 @@ import java.util.concurrent.{Executors, ExecutorService}
class OptimizeTest extends SpecsMatchers with ScalaCheckMatchers {
- //We test this because the optimizer screwed this up once
- @Test
- def testTermFiltersOptimize {
- val q = ESimplePanda where (_.hugenums contains 1L) filter(_.termsfield in List("termhit", "randomterm"))
- val res1 = q fetch()
- val res2 = q optimize() fetch()
- Assert.assertEquals(res1.response.results.length, res2.response.results.length)
- }
-
- @Test
- def testTermFiltersMetallFilter {
- val q = ESimpleGeoPanda where (_.name contains "ordertest") filter(_.metall any)
- val q2 = ESimpleGeoPanda where (_.name contains "ordertest")
- val res1 = q fetch()
- val res2 = q optimize() fetch()
- val res3 = q2 fetch()
- Assert.assertEquals(res1.response.results.length, res2.response.results.length)
- Assert.assertEquals(res1.response.results.length, res3.response.results.length)
- Assert.assertEquals(q.optimize(),q2)
- }
-
@Test
def testProduceCorrectListfieldFilterAny {
val q = SVenueTest where (_.metall any) filter (_.metall any)
@@ -61,14 +40,5 @@ class OptimizeTest extends SpecsMatchers with ScalaCheckMatchers {
"rows" -> "10").sortWith(_._1 > _._1))
}
- @Before
- def esSetup {
- ElasticQueryTest.hoboPrepIndex()
- }
-
- @After
- def esDone {
- ElasticQueryTest.hoboDone()
- }
}

0 comments on commit 721bbae

Please sign in to comment.