Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

CAPEDWARF-31 Search API - Split GAEQueryTreeVisitor into two, Fix neg…

…ation handling, ...
  • Loading branch information...
commit ae9d1f2f7dd2b91805bfb03c24ed67cf9ed8e7d2 1 parent a21d97d
@luksa luksa authored
View
13 pom.xml
@@ -48,6 +48,7 @@
<version.com.google.gae.api>1.6.6</version.com.google.gae.api>
<version.com.google.guava>11.0.2</version.com.google.guava>
<version.json>20090211</version.json>
+ <version.org.apache.lucene>3.6.0</version.org.apache.lucene>
<version.org.apache.velocity>1.6.3</version.org.apache.velocity>
<version.org.bouncycastle>1.47</version.org.bouncycastle>
<version.org.jboss.logging>3.1.1.GA</version.org.jboss.logging>
@@ -353,6 +354,18 @@
</dependency>
<dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>${version.org.apache.lucene}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers</artifactId>
+ <version>${version.org.apache.lucene}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.velocity</groupId>
<artifactId>velocity</artifactId>
<version>${version.org.apache.velocity}</version>
View
10 search/pom.xml
@@ -44,6 +44,16 @@
</dependency>
<dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>com.google.appengine</groupId>
<artifactId>appengine-api-1.0-sdk</artifactId>
</dependency>
View
18 search/src/main/java/org/jboss/capedwarf/search/CapedwarfSearchIndex.java
@@ -29,6 +29,7 @@
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
+import java.util.logging.Logger;
import com.google.appengine.api.search.AddResponse;
import com.google.appengine.api.search.Consistency;
@@ -44,11 +45,6 @@
import com.google.appengine.api.search.Schema;
import com.google.appengine.api.search.ScoredDocument;
import com.google.appengine.api.search.StatusCode;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.queryParser.ParseException;
-import org.apache.lucene.queryParser.QueryParser;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.util.Version;
import org.hibernate.search.query.dsl.QueryBuilder;
import org.hibernate.search.query.dsl.RangeTerminationExcludable;
import org.infinispan.Cache;
@@ -64,6 +60,8 @@
*/
public class CapedwarfSearchIndex implements Index {
+ private final Logger log = Logger.getLogger(getClass().getName());
+
private String name;
private String namespace;
private Consistency consistency;
@@ -232,7 +230,15 @@ private ScoredDocument createScoredDocument(Document document) {
}
private org.apache.lucene.search.Query createLuceneQuery(Query query) {
- return new QueryConverter(CacheValue.ALL_FIELD_NAME).convert(query.getQueryString());
+ QueryConverter queryConverter = new QueryConverter(CacheValue.ALL_FIELD_NAME) {
+ @Override
+ protected GAEQueryTreeVisitor createTreeVisitor() {
+ return new MultiFieldGAEQueryTreeVisitor();
+ }
+ };
+ org.apache.lucene.search.Query luceneQuery = queryConverter.convert(query.getQueryString());
+ log.info("luceneQuery = " + luceneQuery);
+ return luceneQuery;
}
@SuppressWarnings("unchecked")
View
2  search/src/main/java/org/jboss/capedwarf/search/Context.java
@@ -61,6 +61,8 @@ protected void setQuery(Query query) {
public abstract void addSubQuery(Query query);
+ public abstract void addNegatedSubQuery(Query query);
+
public void setOperator(Operator operator) {
this.operator = operator;
}
View
48 search/src/main/java/org/jboss/capedwarf/search/GAEQueryTreeVisitor.java
@@ -22,7 +22,6 @@
package org.jboss.capedwarf.search;
-import com.google.appengine.api.search.Field;
import com.google.appengine.api.search.query.QueryLexer;
import com.google.appengine.api.search.query.QueryTreeVisitor;
import com.google.appengine.api.search.query.QueryTreeWalker;
@@ -44,8 +43,6 @@
*/
public class GAEQueryTreeVisitor implements QueryTreeVisitor<Context> {
- private FieldNamePrefixer fieldNamePrefixer = new FieldNamePrefixer();
-
public static final Version LUCENE_VERSION = Version.LUCENE_35;
public void visitSequence(QueryTreeWalker<Context> walker, Tree tree, Context context) {
@@ -59,6 +56,11 @@ public void visitConjunction(QueryTreeWalker<Context> walker, Tree tree, Context
public void addSubQuery(Query query) {
((BooleanQuery) getQuery()).add(query, BooleanClause.Occur.MUST);
}
+
+ @Override
+ public void addNegatedSubQuery(Query query) {
+ ((BooleanQuery) getQuery()).add(query, BooleanClause.Occur.MUST_NOT);
+ }
};
walkThroughChildren(walker, tree, childContext);
}
@@ -70,17 +72,25 @@ public void visitDisjunction(QueryTreeWalker<Context> walker, Tree tree, Context
public void addSubQuery(Query query) {
((BooleanQuery) getQuery()).add(query, BooleanClause.Occur.SHOULD);
}
+
+ @Override
+ public void addNegatedSubQuery(Query query) {
+ ((BooleanQuery) getQuery()).add(query, BooleanClause.Occur.MUST_NOT);
+ }
};
walkThroughChildren(walker, tree, childContext);
}
- public void visitNegation(QueryTreeWalker<Context> walker, Tree tree, Context context) {
- BooleanQuery booleanQuery = new BooleanQuery();
- context.addSubQuery(booleanQuery);
- Context childContext = new Context(booleanQuery, context.getFieldName(), context.getOperator()) {
+ public void visitNegation(QueryTreeWalker<Context> walker, Tree tree, final Context context) {
+ Context childContext = new ForwardingContext(context) {
@Override
public void addSubQuery(Query query) {
- ((BooleanQuery) getQuery()).add(query, BooleanClause.Occur.MUST_NOT);
+ context.addNegatedSubQuery(query);
+ }
+
+ @Override
+ public void addNegatedSubQuery(Query query) {
+ context.addSubQuery(query);
}
};
walker.walk(tree.getChild(0), childContext);
@@ -149,24 +159,19 @@ private Context newChildContext(final Context context, Operator operator) {
}
public void visitValue(QueryTreeWalker<Context> walker, Tree tree, Context context) {
+ String field = context.getFieldName();
+ Operator operator = context.getOperator();
Tree type = tree.getChild(0);
Tree value = tree.getChild(1);
-
- BooleanQuery booleanQuery = new BooleanQuery();
- for (Field.FieldType fieldType : Field.FieldType.values()) {
- booleanQuery.add(createQuery(context, type, value, fieldType), BooleanClause.Occur.SHOULD);
- }
- context.addSubQuery(booleanQuery);
+ context.addSubQuery(createQuery(field, operator, type, value));
}
- private Query createQuery(Context context, Tree type, Tree value, Field.FieldType fieldType) {
- String prefixedFieldName = fieldNamePrefixer.getPrefixedFieldName(context.getFieldName(), fieldType);
- Operator operator = context.getOperator();
+ protected Query createQuery(String field, Operator operator, Tree type, Tree value) {
if (type.getType() == QueryLexer.NUMBER) {
- return createNumericQuery(prefixedFieldName, operator, value);
+ return createNumericQuery(field, operator, value);
} else {
String text = value.getText().toLowerCase();
- return createQuery(prefixedFieldName, operator, text, type);
+ return createQuery(field, operator, text, type);
}
}
@@ -241,5 +246,10 @@ public ForwardingContext(Context context) {
public void addSubQuery(Query query) {
context.addSubQuery(query);
}
+
+ @Override
+ public void addNegatedSubQuery(Query query) {
+ context.addNegatedSubQuery(query);
+ }
}
}
View
48 search/src/main/java/org/jboss/capedwarf/search/MultiFieldGAEQueryTreeVisitor.java
@@ -0,0 +1,48 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * Copyright 2012, Red Hat, Inc., and individual contributors
+ * as indicated by the @author tags. See the copyright.txt file in the
+ * distribution for a full listing of individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+
+package org.jboss.capedwarf.search;
+
+import com.google.appengine.api.search.Field;
+import com.google.appengine.repackaged.org.antlr.runtime.tree.Tree;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Query;
+
+/**
+ * @author <a href="mailto:mluksa@redhat.com">Marko Luksa</a>
+ */
+public class MultiFieldGAEQueryTreeVisitor extends GAEQueryTreeVisitor {
+
+ private FieldNamePrefixer fieldNamePrefixer = new FieldNamePrefixer();
+
+ protected Query createQuery(String field, Operator operator, Tree type, Tree value) {
+ BooleanQuery booleanQuery = new BooleanQuery();
+ for (Field.FieldType fieldType : Field.FieldType.values()) {
+ String prefixedFieldName = fieldNamePrefixer.getPrefixedFieldName(field, fieldType);
+ Query query = super.createQuery(prefixedFieldName, operator, type, value);
+ booleanQuery.add(query, BooleanClause.Occur.SHOULD);
+ }
+ return booleanQuery;
+ }
+
+}
View
15 search/src/main/java/org/jboss/capedwarf/search/QueryConverter.java
@@ -27,6 +27,8 @@
import com.google.appengine.repackaged.org.antlr.runtime.RecognitionException;
import com.google.appengine.repackaged.org.antlr.runtime.tree.CommonTree;
import com.google.appengine.repackaged.org.antlr.runtime.tree.Tree;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import java.util.logging.Logger;
@@ -59,14 +61,25 @@ private Query convert(Tree tree) {
public void addSubQuery(Query query) {
setQuery(query);
}
+
+ @Override
+ public void addNegatedSubQuery(Query query) {
+ BooleanQuery booleanQuery = new BooleanQuery();
+ booleanQuery.add(query, BooleanClause.Occur.MUST_NOT);
+ setQuery(booleanQuery);
+ }
};
context.setFieldName(allFieldName);
- new QueryTreeWalker<Context>(new GAEQueryTreeVisitor()).walk(tree, context);
+ new QueryTreeWalker<Context>(createTreeVisitor()).walk(tree, context);
return context.getQuery();
}
+ protected GAEQueryTreeVisitor createTreeVisitor() {
+ return new GAEQueryTreeVisitor();
+ }
+
private Tree parseQuery(String queryString) {
try {
CommonTree tree = new QueryTreeBuilder().parse(queryString);
View
25 search/src/test/java/org/jboss/test/capedwarf/search/QueryConverterTestCase.java
@@ -42,7 +42,6 @@
/**
* @author <a href="mailto:mluksa@redhat.com">Marko Luksa</a>
*/
-@Ignore("Should be modified to properly handle prefixed field names")
public class QueryConverterTestCase {
public static final Version LUCENE_VERSION = Version.LUCENE_35;
@@ -55,8 +54,10 @@ public void testConversion() throws Exception {
assertQueryEquals("field:aaa AND field:bbb AND field:ccc", "field:aaa field:bbb field:ccc");
assertQueryEquals("field:aaa OR field:bbb OR field:ccc", "field:aaa OR field:bbb OR field:ccc");
assertQueryEquals("field:aaa AND field:bbb AND field:ccc", "field:aaa AND field:bbb AND field:ccc");
- assertQueryEquals("author:rose OR (NOT body:filigree)", "author:rose OR NOT body:filigree");
- assertQueryEquals("author:rose AND (NOT body:filigree)", "author:rose AND NOT body:filigree");
+ assertQueryEquals("author:rose OR NOT body:filigree", "author:rose OR NOT body:filigree");
+ assertQueryEquals("+author:rose -body:filigree", "author:rose AND NOT body:filigree");
+ assertQueryEquals("+author:rose -(body:filigree author:jones)", "author:rose NOT (body:filigree OR author:jones)");
+ assertQueryEquals("+author:rose -(+body:filigree +author:jones)", "author:rose NOT (body:filigree AND author:jones)");
assertQueryEquals("field:[value TO value]", "field=value");
assertQueryEquals("all:rose", "rose");
@@ -65,11 +66,11 @@ public void testConversion() throws Exception {
assertQueryEquals("bob:hope AND bob:dope", "bob:(hope dope)");
assertQueryEquals("bob:[hope TO hope] OR bob:[dope TO dope]", "bob=(hope OR dope)");
- assertQueryEquals("field:[12 TO 12]", "field=12");
- assertQueryEquals("field:{12 TO *}", "field>12");
- assertQueryEquals("field:[12 TO *]", "field>=12");
- assertQueryEquals("field:{* TO 12}", "field<12");
- assertQueryEquals("field:[* TO 12]", "field<=12");
+ assertQueryEquals("field:[12.0 TO 12.0]", "field=12");
+ assertQueryEquals("field:{12.0 TO *}", "field>12");
+ assertQueryEquals("field:[12.0 TO *]", "field>=12");
+ assertQueryEquals("field:{* TO 12.0}", "field<12");
+ assertQueryEquals("field:[* TO 12.0]", "field<=12");
assertQueryEquals("field:{aaa TO *}", "field>aaa");
assertQueryEquals("field:[aaa TO *]", "field>=aaa");
@@ -95,13 +96,13 @@ public void testExamplesFromGAEDocumentation() throws Exception {
assertQueryEquals("author:rose", "author:rose");
assertQueryEquals("body:\"any other name\"", "body:\"any other name\"");
assertQueryEquals("author:\"Rose Jones\" AND body:rose", "author:\"Rose Jones\" body:rose");
- assertQueryEquals("price:{* TO 100}", "price<100");
+ assertQueryEquals("price:{* TO 100.0}", "price<100");
assertQueryEquals("sent:[2011-02-28 TO *]", "sent>=2011-02-28");
assertQueryEquals("product_code:[xyz1000 TO xyz1000]", "product_code = xyz1000");
assertQueryEquals("author:bob OR ((author:rose OR author:tom) AND author:jones)", "author:(bob OR ((rose OR tom) AND jones))");
- assertQueryEquals("author:rose AND (NOT body:filigree)", "author:rose NOT body:filigree");
- assertQueryEquals("(author:Thomas OR author:Jones) AND (NOT body:rose)", "(author:Thomas OR author:Jones) AND (NOT body:rose)");
+ assertQueryEquals("author:rose AND NOT body:filigree", "author:rose NOT body:filigree");
+ assertQueryEquals("(author:Thomas OR author:Jones) AND NOT body:rose", "(author:Thomas OR author:Jones) AND (NOT body:rose)");
}
private static void assertQueryEquals(String expectedLuceneQueryString, String gaeQueryString) throws ParseException {
@@ -110,7 +111,7 @@ private static void assertQueryEquals(String expectedLuceneQueryString, String g
Query expectedQuery = new QueryParser(LUCENE_VERSION, null, new StandardAnalyzer(LUCENE_VERSION)).parse(expectedLuceneQueryString);
System.out.println("expectedQuery = " + expectedQuery + " (" + expectedQuery.getClass() + ")");
System.out.println("query = " + query + " (" + query.getClass() + ")");
- Assert.assertEquals(expectedQuery, query);
+ Assert.assertEquals(expectedQuery.toString(), query.toString());
}
View
64 search/src/test/java/org/jboss/test/capedwarf/search/SearchTestCase.java
@@ -68,6 +68,16 @@ public void testSearchByTwoFields() {
}
@Test
+ public void testSearchDisjunction() {
+ Index index = getTestIndex();
+ index.add(newDocument("fooaaa", newField("foo").setText("aaa"), newField("bar").setText("bbb")));
+ index.add(newDocument("foobbb", newField("foo").setText("bbb"), newField("bar").setText("ccc")));
+ index.add(newDocument("fooccc", newField("foo").setText("ccc"), newField("bar").setText("bbb")));
+
+ assertSearchYields(index, "foo:aaa OR bar:bbb", "fooaaa", "fooccc");
+ }
+
+ @Test
public void testSearchByTerm() {
Index index = getTestIndex();
index.add(newDocument("fooaaa", newField("foo").setText("bar aaa baz")));
@@ -149,7 +159,7 @@ public void testSearchForNumberInText() {
private Date createDate(int year, int month, int day) {
Calendar cal = Calendar.getInstance();
- cal.set(year, month-1, day, 0, 0, 0);
+ cal.set(year, month - 1, day, 0, 0, 0);
cal.clear(Calendar.MILLISECOND);
return cal.getTime();
}
@@ -159,12 +169,13 @@ private void assertSearchYields(Index index, String queryString, String... docum
Collection<ScoredDocument> scoredDocuments = results.getResults();
System.out.println("-------------------------------");
System.out.println("queryString = " + queryString);
+ System.out.println("scoredDocuments = " + scoredDocuments);
for (ScoredDocument scoredDocument : scoredDocuments) {
System.out.println("scoredDocument = " + scoredDocument);
}
- assertEquals(documentIds.length, results.getNumberFound());
- assertEquals(documentIds.length, results.getNumberReturned());
- assertEquals(documentIds.length, scoredDocuments.size());
+ assertEquals("number of found documents", documentIds.length, results.getNumberFound());
+ assertEquals("number of returned documents", documentIds.length, results.getNumberReturned());
+ assertEquals("actual number of ScoredDcuments", documentIds.length, scoredDocuments.size());
Set<String> expectedDocumentIds = new HashSet<String>(Arrays.asList(documentIds));
for (ScoredDocument scoredDocument : scoredDocuments) {
@@ -217,4 +228,49 @@ public void testSearchOnAllFields() {
assertEquals(2, index.search("aaa").getResults().size());
}
+ @Test
+ public void testComplexSearch1() {
+ Index index = getTestIndex();
+ index.add(newDocument("bm", newField("author").setText("Bob Marley")));
+ index.add(newDocument("rj", newField("author").setText("Rose Jones")));
+ index.add(newDocument("rt", newField("author").setText("Rose Trunk")));
+ index.add(newDocument("tj", newField("author").setText("Tom Jones")));
+
+ assertSearchYields(index, "author:(bob OR ((rose OR tom) AND jones))", "bm", "rj", "tj");
+ }
+
+ @Test
+ public void testSearchWithNegation() {
+
+ Index index = getTestIndex();
+ index.add(newDocument("with_baz", newField("body").setText("Foo bar baz")));
+ index.add(newDocument("without_baz", newField("body").setText("Foo bar.")));
+
+ assertSearchYields(index, "body:foo AND NOT body:baz", "without_baz");
+ assertSearchYields(index, "body:foo NOT body:baz", "without_baz");
+ }
+
+ @Ignore("check if this is even a valid test")
+ @Test
+ public void testSearchWithNegation2() {
+
+ Index index = getTestIndex();
+ index.add(newDocument("foo_with_baz", newField("body").setText("Foo bar baz")));
+ index.add(newDocument("foo_without_baz", newField("body").setText("Foo bar.")));
+ index.add(newDocument("without_foo_without_baz", newField("body").setText("bar.")));
+ index.add(newDocument("without_foo_with_baz", newField("body").setText("bar baz.")));
+
+ assertSearchYields(index, "body:foo OR NOT body:baz", "foo_with_baz", "foo_without_baz", "without_foo_without_baz");
+ }
+
+ @Ignore("Not supported by Lucene. Must check if it is even supported by GAE")
+ @Test
+ public void testSearchWithNegationOnly() {
+ Index index = getTestIndex();
+ index.add(newDocument("with_baz", newField("body").setText("Foo bar baz")));
+ index.add(newDocument("without_baz", newField("body").setText("Foo bar.")));
+
+ assertSearchYields(index, "NOT body=baz", "without_baz");
+ }
+
}
Please sign in to comment.
Something went wrong with that request. Please try again.