Permalink
Browse files

Merge remote branch 'jw/master' into rmaster

  • Loading branch information...
2 parents 9d9f7da + 12c3d6b commit 7adcb67a89da6d7dd91251095a3321997c72061d @wonlay wonlay committed Dec 14, 2011
@@ -382,6 +382,7 @@ public void run()
}
}
}
+ flush();
}
private long getEventCount()
@@ -33,13 +33,15 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.Similarity;
import proj.zoie.api.DataConsumer;
import proj.zoie.api.ZoieException;
import proj.zoie.api.ZoieHealth;
+import proj.zoie.api.ZoieIndexReader;
import proj.zoie.api.ZoieSegmentReader;
import proj.zoie.api.indexing.AbstractZoieIndexable;
import proj.zoie.api.indexing.IndexingEventListener;
@@ -78,26 +80,29 @@ public void setPurgeFilter(Filter purgeFilter){
private final void purgeDocuments(){
if (_purgeFilter!=null){
BaseSearchIndex<R> idx = getSearchIndex();
- IndexReader reader = null;
+ IndexReader writeReader = null;
log.info("purging docs started...");
int count = 0;
long start = System.currentTimeMillis();
try{
- reader = idx.openIndexReaderForDelete();
+ writeReader = idx.openIndexReaderForDelete();
+
+ ZoieIndexReader<R> reader = idx.openIndexReader();
DocIdSetIterator iter = _purgeFilter.getDocIdSet(reader).iterator();
+
int doc;
while((doc = iter.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS){
count++;
- reader.deleteDocument(doc);
+ writeReader.deleteDocument(doc);
}
}
catch(Throwable th){
log.error("problem creating purge filter: "+th.getMessage(),th);
}
finally{
- if (reader!=null){
+ if (writeReader!=null){
try{
- reader.close();
+ writeReader.close();
}
catch(IOException ioe){
ZoieHealth.setFatal();
@@ -26,7 +26,9 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Filter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
@@ -259,13 +261,63 @@ public void testRealtime2() throws ZoieException {
}
}
+ private static class EvenIDPurgeFilter extends Filter{
+
+ @Override
+ public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
+ if (reader instanceof ZoieIndexReader){
+ final ZoieIndexReader<IndexReader> zoieReader = (ZoieIndexReader<IndexReader>)reader;
+ return new DocIdSet(){
+
+ @Override
+ public DocIdSetIterator iterator() throws IOException {
+ return new DocIdSetIterator(){
+
+ int doc=-1;
+ int maxdoc = zoieReader.maxDoc();
+
+ @Override
+ public int advance(int target) throws IOException {
+ doc = target-1;
+ return nextDoc();
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ while(doc<maxdoc){
+ doc++;
+ long uid = zoieReader.getUID(doc);
+ if (uid %2 == 0){ // if even
+ return doc;
+ }
+ }
+ return DocIdSetIterator.NO_MORE_DOCS;
+ }
+
+ };
+ }
+
+ };
+ }
+ else{
+ throw new IllegalStateException("expecting instance of ZoieIndexReader, but got: "+reader.getClass());
+ }
+ }
+
+ }
+
@Test
public void testPurgeFilter() throws Exception {
File idxDir = getIdxDir();
ZoieSystem<IndexReader, String> idxSystem = createZoie(
idxDir, true, ZoieConfig.DEFAULT_VERSION_COMPARATOR,true);
- idxSystem.setPurgeFilter(new QueryWrapperFilter(new MatchAllDocsQuery()));
+ idxSystem.setPurgeFilter(new EvenIDPurgeFilter());
idxSystem.start();
MemoryStreamDataProvider<String> memoryProvider = new MemoryStreamDataProvider<String>(ZoieConfig.DEFAULT_VERSION_COMPARATOR);
@@ -321,7 +373,7 @@ public void testPurgeFilter() throws Exception {
numDocs = multiReader.numDocs();
log.info("new numdocs: "+numDocs);
- TestCase.assertTrue(numDocs==0);
+ TestCase.assertTrue(numDocs==5);
idxSystem.returnIndexReaders(readers);
@@ -27,7 +27,7 @@
public LinedFileDataProvider(File file,long startingOffset){
- super(ZoieConfig.DEFAULT_VERSION_COMPARATOR);
+ super(ZoiePerfVersion.COMPARATOR);
_file = file;
_rad = null;
_startingOffset = startingOffset;
@@ -43,15 +43,15 @@ public LinedFileDataProvider(File file,long startingOffset){
if (line == null) return null;
String version = ZoiePerfVersion.toString(_count,_offset);
- _offset+=version.length();
+ _offset+=line.length();
+ _count++;
event = new DataEvent<String>(line,version);
}
catch(IOException ioe){
logger.error(ioe.getMessage(),ioe);
}
}
- _count++;
return event;
}

0 comments on commit 7adcb67

Please sign in to comment.