Permalink
Browse files

Enhanced unit tests for the AttributeFacetHandler

  • Loading branch information...
Volodymyr Zhabiuk
Volodymyr Zhabiuk committed Jan 11, 2012
2 parents 757a440 + 19af807 commit 559dbfa2d000f7446604943633bf82cdc17c8722
Showing with 31,039 additions and 16 deletions.
  1. +4 −1 sensei-core/src/main/java/com/sensei/conf/SenseiFacetHandlerBuilder.java
  2. +108 −0 sensei-core/src/main/java/com/sensei/facet/attribute/AttributesFacetCountCollector.java
  3. +88 −0 sensei-core/src/main/java/com/sensei/facet/attribute/AttributesFacetHandler.java
  4. +56 −0 sensei-core/src/main/java/com/sensei/facet/attribute/AttributesFacetIterator.java
  5. +12 −0 sensei-core/src/main/java/com/sensei/facet/attribute/FacetPredicate.java
  6. +60 −0 sensei-core/src/main/java/com/sensei/facet/attribute/PredicateDocIdIterator.java
  7. +75 −0 sensei-core/src/main/java/com/sensei/facet/attribute/PredicateFacetFilter.java
  8. +34 −0 sensei-core/src/main/java/com/sensei/facet/attribute/Range.java
  9. +142 −0 sensei-core/src/main/java/com/sensei/facet/attribute/RangePredicate.java
  10. +19 −0 sensei-core/src/main/java/com/sensei/facet/attribute/SimpleDataCacheBuilder.java
  11. +287 −0 sensei-core/src/test/java/com/sensei/facet/attributes/AttributesFacetHandlerTest.java
  12. +7 −11 sensei-core/src/test/java/com/sensei/test/SenseiStarter.java
  13. +1 −1 sensei-core/src/test/java/com/sensei/test/TestSensei.java
  14. +131 −0 sensei-core/src/test/java/com/sensei/test/TestSenseiAttributesHandler.java
  15. +30,000 −0 sensei-core/src/test/resources/data/test_data.json
  16. 0 sensei-core/src/test/resources/{conf → test-conf}/bobo.spring
  17. +6 −1 sensei-core/src/test/resources/{conf → test-conf}/node1/schema.xml
  18. +1 −1 sensei-core/src/test/resources/{conf → test-conf}/node1/sensei.properties
  19. +7 −0 sensei-core/src/test/resources/{conf → test-conf}/node2/schema.xml
  20. +1 −1 sensei-core/src/test/resources/{conf → test-conf}/node2/sensei.properties
  21. 0 sensei-core/src/test/resources/{conf → test-conf}/sensei-test.spring
  22. 0 sensei-core/src/test/resources/{conf → test-conf}/zoie.spring
@@ -31,6 +31,7 @@
import com.browseengine.bobo.facets.impl.PathFacetHandler;
import com.browseengine.bobo.facets.impl.RangeFacetHandler;
import com.browseengine.bobo.facets.impl.SimpleFacetHandler;
+import com.sensei.facet.attribute.AttributesFacetHandler;
import com.sensei.indexing.api.DefaultSenseiInterpreter;
import com.sensei.plugin.SenseiPluginRegistry;
import com.sensei.search.facet.UIDFacetHandler;
@@ -392,7 +393,9 @@ public static SenseiSystemInfo buildFacets(JSONObject schemaObj, SenseiPluginReg
facetHandler = buildMultiHandler(name, fieldName, termListFactoryMap.get(fieldName), dependSet);
} else if (type.equals("compact-multi")) {
facetHandler = buildCompactMultiHandler(name, fieldName, dependSet, termListFactoryMap.get(fieldName));
- } else if (type.equals("histogram")) {
+ } else if (type.equals("attribute")) {
+ facetHandler = new AttributesFacetHandler(name, fieldName, termListFactoryMap.get(fieldName), null , dependSet, facetProps);
+ } else if (type.equals("histogram")) {
// A histogram facet handler is always dynamic
RuntimeFacetHandlerFactory<?, ?> runtimeFacetFactory = getHistogramFacetHandlerFactory(facet, name, paramMap);
runtimeFacets.add(runtimeFacetFactory);
@@ -0,0 +1,108 @@
+package com.sensei.facet.attribute;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.browseengine.bobo.api.BrowseFacet;
+import com.browseengine.bobo.api.BrowseSelection;
+import com.browseengine.bobo.api.FacetIterator;
+import com.browseengine.bobo.api.FacetSpec;
+import com.browseengine.bobo.facets.data.MultiValueFacetDataCache;
+import com.browseengine.bobo.facets.impl.DefaultFacetCountCollector;
+import com.browseengine.bobo.util.BigNestedIntArray;
+
+public final class AttributesFacetCountCollector extends DefaultFacetCountCollector {
+ private final AttributesFacetHandler attributesFacetHandler;
+ public final BigNestedIntArray _array;
+ private RangePredicate rangePredicate;
+ private int[] buffer;
+ private List<BrowseFacet> cachedFacets;
+ private final int numFacetsPerKey;
+ private final char separator;
+ @SuppressWarnings("rawtypes")
+ public AttributesFacetCountCollector(AttributesFacetHandler attributesFacetHandler, String name, MultiValueFacetDataCache dataCache, int docBase, BrowseSelection browseSelection, FacetSpec ospec, int numFacetsPerKey, char separator){
+ super(name,dataCache,docBase,browseSelection,ospec);
+ this.attributesFacetHandler = attributesFacetHandler;
+ this.numFacetsPerKey = numFacetsPerKey;
+ this.separator = separator;
+ _array = dataCache._nestedArray;
+ buffer = new int[10];
+ if (browseSelection != null) {
+ rangePredicate = new RangePredicate(browseSelection.getValues(), browseSelection.getNotValues(), browseSelection.getSelectionOperation(), separator);
+ }
+ }
+
+ @Override
+ public final void collect(int docid) {
+ if (rangePredicate != null) {
+ if (buffer.length < _array.getNumItems(docid)) {
+ buffer = new int[_array.getNumItems(docid) + 10];
+ }
+ int count = _array.getData(docid, buffer);
+ if (count == 1) {
+ if (rangePredicate.evaluateValue(_dataCache, buffer[0])) {
+ _count[buffer[0]]++;
+ }
+ }
+ for (int i = 0; i < count; i++) {
+ if (rangePredicate.evaluateValue(_dataCache, buffer[i])) {
+ _count[buffer[i]]++;
+ }
+ }
+ } else {
+ _array.countNoReturn(docid, _count);
+ }
+ }
+
+ @Override
+ public final void collectAll()
+ {
+ _count = _dataCache.freqs;
+ }
+ @Override
+ public List<BrowseFacet> getFacets() {
+ if (cachedFacets == null) {
+ int max = _ospec.getMaxCount();
+ _ospec.setMaxCount(-1);
+ List<BrowseFacet> facets = super.getFacets();
+ _ospec.setMaxCount(max);
+ filterByKeys(facets, separator, numFacetsPerKey);
+ cachedFacets = facets;
+ }
+ return cachedFacets;
+ }
+
+ private void filterByKeys(List<BrowseFacet> facets, char separator, int numFacetsPerKey) {
+ Map<String, AtomicInteger> keyOccurences = new HashMap<String, AtomicInteger>();
+ Iterator<BrowseFacet> iterator = facets.iterator();
+ String separatorString = String.valueOf(separator);
+ while (iterator.hasNext()) {
+ BrowseFacet facet = iterator.next();
+ String value = facet.getValue();
+ if (!value.contains(separatorString)) {
+ iterator.remove();
+ continue;
+ }
+ String key = value.substring(0, value.indexOf(separatorString));
+ AtomicInteger numOfKeys = keyOccurences.get(key);
+ if (numOfKeys == null) {
+ numOfKeys = new AtomicInteger(0);
+ keyOccurences.put(key, numOfKeys);
+ }
+ int count = numOfKeys.incrementAndGet();
+ if (count > numFacetsPerKey) {
+ iterator.remove();
+ }
+ }
+
+ }
+
+ @Override
+ public FacetIterator iterator() {
+ return new AttributesFacetIterator(getFacets());
+ }
+
+}
@@ -0,0 +1,88 @@
+package com.sensei.facet.attribute;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.lucene.index.Term;
+
+import com.browseengine.bobo.api.BoboIndexReader;
+import com.browseengine.bobo.api.BrowseSelection;
+import com.browseengine.bobo.api.BrowseSelection.ValueOperation;
+import com.browseengine.bobo.api.FacetSpec;
+import com.browseengine.bobo.facets.FacetCountCollector;
+import com.browseengine.bobo.facets.FacetCountCollectorSource;
+import com.browseengine.bobo.facets.data.MultiValueFacetDataCache;
+import com.browseengine.bobo.facets.data.TermListFactory;
+import com.browseengine.bobo.facets.filter.EmptyFilter;
+import com.browseengine.bobo.facets.filter.RandomAccessFilter;
+import com.browseengine.bobo.facets.filter.RandomAccessNotFilter;
+import com.browseengine.bobo.facets.impl.MultiValueFacetHandler;
+
+public class AttributesFacetHandler extends MultiValueFacetHandler {
+ public static final char DEFAULT_SEPARATOR = '=';
+ private char separator;
+ private int numFacetsPerKey = 7;
+ public static final String SEPARATOR_PROP_NAME = "separator";
+ public static final String NUM_FACETS_PER_KEY_PROP_NAME = "numFacetsPerKey";
+
+ public AttributesFacetHandler(String name, String indexFieldName, TermListFactory termListFactory, Term sizePayloadTerm, Set<String> depends, Map<String, String> facetProps) {
+ super(name, indexFieldName, termListFactory, sizePayloadTerm, depends);
+ if (facetProps.containsKey(SEPARATOR_PROP_NAME)) {
+ this.separator = narrow(facetProps.get(SEPARATOR_PROP_NAME)).charAt(0);
+ } else {
+ this.separator = DEFAULT_SEPARATOR;
+ }
+ if (facetProps.containsKey(NUM_FACETS_PER_KEY_PROP_NAME)) {
+ this.numFacetsPerKey = Integer.parseInt(narrow(facetProps.get(NUM_FACETS_PER_KEY_PROP_NAME)));
+ }
+ }
+ private String narrow(String string) {
+ return string.replaceAll("\\[", "").replaceAll("\\]", "");
+ }
+ public char getSeparator(BrowseSelection browseSelection) {
+ if (browseSelection == null || !browseSelection.getSelectionProperties().containsKey(SEPARATOR_PROP_NAME)) {
+ return separator;
+ }
+ return browseSelection.getSelectionProperties().get(SEPARATOR_PROP_NAME).toString().charAt(0);
+ }
+ public int getFacetsPerKey(BrowseSelection browseSelection) {
+ if (browseSelection == null || !browseSelection.getSelectionProperties().containsKey(NUM_FACETS_PER_KEY_PROP_NAME)) {
+ return numFacetsPerKey;
+ }
+ return Integer.valueOf(browseSelection.getSelectionProperties().get(NUM_FACETS_PER_KEY_PROP_NAME).toString());
+ }
+ @Override
+ public RandomAccessFilter buildFilter(final BrowseSelection browseSelection) throws IOException {
+ final String[] values = browseSelection.getValues();
+ final String[] notValues = browseSelection.getNotValues();
+ final ValueOperation operation = browseSelection.getSelectionOperation();
+
+ if (values.length ==0 && notValues.length == 0) {
+ return EmptyFilter.getInstance();
+ } else if (values.length ==0 && notValues.length > 0) {
+ return new RandomAccessNotFilter(new PredicateFacetFilter(new SimpleDataCacheBuilder(getName()), new RangePredicate(notValues, values, operation, getSeparator(browseSelection))));
+ } else return new PredicateFacetFilter(new SimpleDataCacheBuilder(getName()), new RangePredicate(values, notValues, operation, getSeparator(browseSelection)));
+ }
+ @Override
+ public RandomAccessFilter buildRandomAccessAndFilter(String[] vals, Properties prop) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+ @Override
+ public RandomAccessFilter buildRandomAccessOrFilter(String[] vals, Properties prop, boolean isNot) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+ @Override
+ public FacetCountCollectorSource getFacetCountCollectorSource(final BrowseSelection browseSelection, final FacetSpec ospec){
+ return new FacetCountCollectorSource(){
+
+ @Override
+ public FacetCountCollector getFacetCountCollector(
+ BoboIndexReader reader, int docBase) {
+ MultiValueFacetDataCache dataCache = (MultiValueFacetDataCache) reader.getFacetData(_name);
+ return new AttributesFacetCountCollector(AttributesFacetHandler.this, _name,dataCache,docBase,browseSelection, ospec, getFacetsPerKey(browseSelection), getSeparator(browseSelection));
+ }
+ };
+ }
+}
@@ -0,0 +1,56 @@
+package com.sensei.facet.attribute;
+
+import java.util.Iterator;
+import java.util.List;
+
+import com.browseengine.bobo.api.BrowseFacet;
+import com.browseengine.bobo.api.FacetIterator;
+
+public class AttributesFacetIterator extends FacetIterator {
+ private Iterator<BrowseFacet> iterator;
+
+ public AttributesFacetIterator(List<BrowseFacet> facets) {
+ iterator = facets.iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ // TODO Auto-generated method stub
+ return iterator.hasNext();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+
+ }
+
+ @Override
+ public Comparable next() {
+ count = 0;
+ BrowseFacet next = iterator.next();
+ if (next == null) {
+ return null;
+ }
+ facet = next.getValue();
+ return next.getValue();
+ }
+
+ @Override
+ public Comparable next(int minHits) {
+ while (iterator.hasNext()) {
+ BrowseFacet next = iterator.next();
+ count = next.getFacetValueHitCount();
+ facet = next.getValue();
+ if (next.getFacetValueHitCount() >= minHits) {
+ return next.getValue();
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public String format(Object val) {
+ return val != null ? val.toString() : null;
+ }
+}
@@ -0,0 +1,12 @@
+package com.sensei.facet.attribute;
+
+import com.browseengine.bobo.facets.data.FacetDataCache;
+
+public interface FacetPredicate {
+ public boolean evaluate(FacetDataCache cache, int docId);
+ public boolean evaluateValue(FacetDataCache cache, int valId);
+ public int valueStartIndex(FacetDataCache cache);
+ public int valueEndIndex(FacetDataCache cache);
+
+
+}
@@ -0,0 +1,60 @@
+package com.sensei.facet.attribute;
+
+import java.io.IOException;
+
+import org.apache.lucene.search.DocIdSetIterator;
+
+import com.browseengine.bobo.facets.data.FacetDataCache;
+@SuppressWarnings("rawtypes")
+public class PredicateDocIdIterator extends DocIdSetIterator {
+ private final int startDocId;
+ private final int endDocId;
+ private final FacetPredicate facetPredicate;
+ private int docId = -1;
+ private final FacetDataCache facetDataCache;
+
+
+ public PredicateDocIdIterator(int startDocId, int endDocId, FacetPredicate facetPredicate, FacetDataCache facetDataCache) {
+
+ this.startDocId = startDocId;
+ this.endDocId = endDocId;
+ if (startDocId > endDocId) {
+ throw new IllegalStateException("startID shouldn't be greater than endId");
+ }
+ this.facetPredicate = facetPredicate;
+ this.facetDataCache = facetDataCache;
+ }
+
+ @Override
+ public int docID() {
+ return docId;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ if (docId == -1) {
+ docId = startDocId - 1;
+ }
+ if (docId == NO_MORE_DOCS) {
+ return docId;
+ }
+ docId++;
+ while (!facetPredicate.evaluate(facetDataCache, docId) || docId > endDocId) {
+ if (docId > endDocId) {
+ docId = NO_MORE_DOCS;
+ return docId;
+ }
+ docId++;
+ }
+ return docId;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (target > endDocId) {
+ return NO_MORE_DOCS;
+ }
+ docId = target - 1;
+ return nextDoc();
+ }
+}
Oops, something went wrong.

0 comments on commit 559dbfa

Please sign in to comment.