Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

added zoie purge support

  • Loading branch information...
commit c77f36aab12e10507357765095a61bf2e2783ccc 1 parent d73b6cf
@javasoze javasoze authored
View
7 conf/sensei.properties
@@ -9,7 +9,7 @@ sensei.server.requestThreadMaxPoolSize=70
sensei.server.requestThreadKeepAliveTimeSecs=300
# sensei cluster parameters
-sensei.cluster.name=sensei
+sensei.cluster.name=sensei-car-demo
sensei.cluster.url=localhost:2181
sensei.cluster.timeout=30000
@@ -39,6 +39,9 @@ sensei.index.manager.default.file.path = data/cars.json
# indexer type, zoie/hourglass/<custom name>
sensei.indexer.type=zoie
+#sensei.indexer.zoie.retention.days=
+#sensei.indexer.zoie.retention.field=
+#sensei.indexer.zoie.retention.timeunit=
#extra parameters for hourglass
@@ -64,7 +67,7 @@ sensei.broker.maxThread = 100
sensei.broker.maxWaittime = 2000
sensei.broker.webapp.path=src/main/webapp
-sensei.search.cluster.name = sensei
+sensei.search.cluster.name = sensei-car-demo
sensei.search.cluster.zookeeper.url = localhost:2181
sensei.search.cluster.zookeeper.conn.timeout = 30000
sensei.search.cluster.network.conn.timeout = 1000
View
9 src/main/java/com/sensei/conf/SenseiConfParams.java
@@ -1,6 +1,9 @@
package com.sensei.conf;
import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
public interface SenseiConfParams {
public static final String NODE_ID = "sensei.node.id";
@@ -46,6 +49,8 @@
public static final String SENSEI_HOURGLASS_FREQUENCY_HOUR = "hour";
public static final String SENSEI_HOURGLASS_FREQUENCY_DAY = "day";
+ public static final String SENSEI_ZOIE_PURGER = "sensei.indexer.zoie.purger";
+
public static final String SERVER_BROKER_PORT = "sensei.broker.port";
public static final String SERVER_BROKER_WEBAPP_PATH = "sensei.broker.webapp.path";
@@ -58,7 +63,9 @@
public static final String SENSEI_INDEX_PRUNER = "sensei.index.pruner";
- public static final String SENSEI_ZOIE_RETENTION_DAYS = "sensei.index.zoie.retentionDays";
+ public static final String SENSEI_ZOIE_RETENTION_DAYS = "sensei.indexer.zoie.retention.days";
+ public static final String SENSEI_ZOIE_RETENTION_FIELD = "sensei.indexer.zoie.retention.field";
+ public static final String SENSEI_ZOIE_RETENTION_TIMEUNIT = "sensei.indexer.zoie.retention.timeunit";
public static final String SENSEI_MX4J_PORT = "sensei.mx4j.port";
View
2  src/main/java/com/sensei/conf/SenseiFacetHandlerBuilder.java
@@ -1,7 +1,5 @@
package com.sensei.conf;
-import java.util.ArrayList;
-import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
View
21 src/main/java/com/sensei/conf/SenseiServerBuilder.java
@@ -11,6 +11,7 @@
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.Map;
@@ -72,6 +73,7 @@
import com.sensei.search.nodes.SenseiZoieFactory;
import com.sensei.search.nodes.SenseiZoieSystemFactory;
import com.sensei.search.nodes.impl.DefaultJsonQueryBuilderFactory;
+import com.sensei.search.nodes.impl.SenseiTimeFilter;
import com.sensei.search.req.AbstractSenseiRequest;
import com.sensei.search.req.AbstractSenseiResult;
import com.sensei.search.req.SenseiSystemInfo;
@@ -94,6 +96,14 @@
private final Server _jettyServer;
private final Comparator<String> _versionComparator;
+ public static final Map<String,TimeUnit> TimeUnitStringMap = new HashMap<String,TimeUnit>();
+
+ static{
+ TimeUnitStringMap.put("days",TimeUnit.DAYS);
+ TimeUnitStringMap.put("seconds",TimeUnit.SECONDS);
+ TimeUnitStringMap.put("milliseconds",TimeUnit.MILLISECONDS);
+ }
+
private static ApplicationContext loadSpringContext(File confFile){
ApplicationContext springCtx = null;
if (confFile.exists()){
@@ -409,12 +419,21 @@ public SenseiCore buildCore() throws ConfigurationException{
SenseiZoieFactory<?> zoieSystemFactory = null;
if (SENSEI_INDEXER_TYPE_ZOIE.equals(indexerType)){
- zoieSystemFactory = new SenseiZoieSystemFactory(idxDir,interpreter,decorator,
+ SenseiZoieSystemFactory zoieFactory = new SenseiZoieSystemFactory(idxDir,interpreter,decorator,
zoieConfig);
+ zoieSystemFactory = zoieFactory;
int retentionDays = _senseiConf.getInt(SENSEI_ZOIE_RETENTION_DAYS,-1);
if (retentionDays>0){
+ String retentionField=_senseiConf.getString(SENSEI_ZOIE_RETENTION_FIELD);
+ String retentionTimeUnit=_senseiConf.getString(SENSEI_ZOIE_RETENTION_TIMEUNIT).toLowerCase();
+ TimeUnit timeUnit = TimeUnitStringMap.get(retentionTimeUnit);
+ if (timeUnit==null){
+ throw new ConfigurationException("invalid timeunit: "+timeUnit+", only days/seconds/milliseconds supported");
+ }
+ SenseiTimeFilter purgeFilter = new SenseiTimeFilter(retentionField, timeUnit, retentionDays);
+ zoieFactory.setPurgeFilter(purgeFilter);
}
}
else if (SENSEI_INDEXER_TYPE_HOURGLASS.equals(indexerType)){
View
10 src/main/java/com/sensei/search/nodes/SenseiZoieSystemFactory.java
@@ -3,6 +3,7 @@
import java.io.File;
import org.apache.log4j.Logger;
+import org.apache.lucene.search.Filter;
import proj.zoie.api.indexing.ZoieIndexableInterpreter;
import proj.zoie.impl.indexing.ZoieConfig;
@@ -14,10 +15,16 @@
{
private static Logger log = Logger.getLogger(SenseiZoieSystemFactory.class);
+ private Filter _purgeFilter;
public SenseiZoieSystemFactory(File idxDir, ZoieIndexableInterpreter<T> interpreter, SenseiIndexReaderDecorator indexReaderDecorator,
ZoieConfig zoieConfig)
{
super(idxDir,interpreter,indexReaderDecorator,zoieConfig);
+ _purgeFilter = null;
+ }
+
+ public void setPurgeFilter(Filter purgeFilter){
+ _purgeFilter = purgeFilter;
}
@Override
@@ -30,6 +37,9 @@ public SenseiZoieSystemFactory(File idxDir, ZoieIndexableInterpreter<T> interpre
log.info("nodeId="+nodeId+", partition=" + partitionId + " does not exist, directory created.");
}
ZoieSystem<BoboIndexReader,T> zoie = new ZoieSystem<BoboIndexReader,T>(partDir, _interpreter, _indexReaderDecorator, _zoieConfig);
+ if (_purgeFilter!=null){
+ zoie.setPurgeFilter(_purgeFilter);
+ }
return zoie;
}
View
60 src/main/java/com/sensei/search/nodes/impl/SenseiTimeFilter.java
@@ -0,0 +1,60 @@
+package com.sensei.search.nodes.impl;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.DocIdSet;
+import org.apache.lucene.search.Filter;
+
+import com.browseengine.bobo.api.BoboIndexReader;
+import com.browseengine.bobo.facets.FacetHandler;
+import com.browseengine.bobo.facets.filter.RandomAccessFilter;
+
+public class SenseiTimeFilter extends Filter {
+
+ private static final long serialVersionUID = 1L;
+ private final String _timeField;
+ private final TimeUnit _timeUnit;
+ private final int _numDays;
+
+ public SenseiTimeFilter(String timeField,TimeUnit timeUnit,int numDays){
+ _timeField = timeField;
+ _timeUnit = timeUnit;
+ _numDays = numDays;
+ }
+
+ public static long buildFromTime(int numDays,TimeUnit timeUnit){
+ long time = System.currentTimeMillis();
+ long convertedNow=timeUnit.convert(time, TimeUnit.MILLISECONDS);
+ long converted = timeUnit.convert(numDays, TimeUnit.DAYS);
+ return convertedNow - converted;
+ }
+
+ private static String buildTimeRangeString(int numDays,TimeUnit timeUnit){
+
+ long from = buildFromTime(numDays,timeUnit);
+ StringBuilder buf = new StringBuilder();
+ buf.append("[").append(from).append(" TO *]");
+ return buf.toString();
+ }
+
+ @Override
+ public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
+ if (reader instanceof BoboIndexReader){
+ BoboIndexReader boboReader = (BoboIndexReader)reader;
+ FacetHandler<?> timeFacetHandler = boboReader.getFacetHandler(_timeField);
+ if (timeFacetHandler==null){
+ throw new IOException(_timeField+" not defined");
+ }
+
+ String rangeString = buildTimeRangeString(_numDays,_timeUnit);
+ RandomAccessFilter filter = timeFacetHandler.buildRandomAccessFilter(rangeString, null);
+
+ return filter.getDocIdSet(reader);
+ }
+ else{
+ throw new IOException("reader not instance of "+BoboIndexReader.class);
+ }
+ }
+}
View
18 src/test/java/com/sensei/test/TestIndexingAPI.java
@@ -8,14 +8,12 @@
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import junit.framework.TestCase;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.Field.TermVector;
import proj.zoie.api.indexing.ZoieIndexable;
import proj.zoie.api.indexing.ZoieIndexable.IndexingReq;
@@ -28,6 +26,7 @@
import com.sensei.indexing.api.StoredValue;
import com.sensei.indexing.api.Text;
import com.sensei.indexing.api.Uid;
+import com.sensei.search.nodes.impl.SenseiTimeFilter;
public class TestIndexingAPI extends TestCase {
@@ -216,6 +215,19 @@ public void testMetaContent(){
}
}
+ public void testRetentionTimeCalculation(){
+ long now = System.currentTimeMillis();
+ long nowDays = TimeUnit.DAYS.convert(now, TimeUnit.MILLISECONDS);
+ int numDays = 13;
+
+ TimeUnit tu = TimeUnit.SECONDS;
+ long from = SenseiTimeFilter.buildFromTime(13,tu);
+
+ long fromDays = TimeUnit.DAYS.convert(from, tu);
+ assertEquals(numDays,nowDays-fromDays);
+
+ }
+
public static void main(String[] args) {
DefaultSenseiInterpreter<TestObj> nodeInterpreter = new DefaultSenseiInterpreter(TestObj.class);
System.out.println(nodeInterpreter);

0 comments on commit c77f36a

Please sign in to comment.
Something went wrong with that request. Please try again.