Merge remote branch 'jw/master' into rmaster

senseidb · Dec 14, 2011 · 7adcb67 · 7adcb67
2 parents 9d9f7da + 12c3d6b
commit 7adcb67
Show file tree

Hide file tree

Showing 4 changed files with 68 additions and 10 deletions.
diff --git a/zoie-core/src/main/java/proj/zoie/impl/indexing/StreamDataProvider.java b/zoie-core/src/main/java/proj/zoie/impl/indexing/StreamDataProvider.java
@@ -382,6 +382,7 @@ public void run()
           }
         }
       }
+      flush();
     }
 
     private long getEventCount()

diff --git a/zoie-core/src/main/java/proj/zoie/impl/indexing/internal/LuceneIndexDataLoader.java b/zoie-core/src/main/java/proj/zoie/impl/indexing/internal/LuceneIndexDataLoader.java
@@ -33,13 +33,15 @@
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.Similarity;
 
 import proj.zoie.api.DataConsumer;
 import proj.zoie.api.ZoieException;
 import proj.zoie.api.ZoieHealth;
+import proj.zoie.api.ZoieIndexReader;
 import proj.zoie.api.ZoieSegmentReader;
 import proj.zoie.api.indexing.AbstractZoieIndexable;
 import proj.zoie.api.indexing.IndexingEventListener;
@@ -78,26 +80,29 @@ public void setPurgeFilter(Filter purgeFilter){
     private final void purgeDocuments(){
     	if (_purgeFilter!=null){
     		BaseSearchIndex<R> idx = getSearchIndex();
-    		IndexReader reader = null;
+    		IndexReader writeReader = null;
     		log.info("purging docs started...");
     		int count = 0;
     		long start = System.currentTimeMillis();
     		try{
-    			reader = idx.openIndexReaderForDelete();
+    		  writeReader = idx.openIndexReaderForDelete();
+
+    			ZoieIndexReader<R> reader = idx.openIndexReader();
     			DocIdSetIterator iter = _purgeFilter.getDocIdSet(reader).iterator();
+
     			int doc;
     			while((doc = iter.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS){
     				count++;
-    				reader.deleteDocument(doc);
+    				writeReader.deleteDocument(doc);
     			}
     		}
     		catch(Throwable th){
     			log.error("problem creating purge filter: "+th.getMessage(),th);
     		}
     		finally{
-    			if (reader!=null){
+    			if (writeReader!=null){
     				try{
-    					reader.close();
+    				  writeReader.close();
     				}
     				catch(IOException ioe){
     					ZoieHealth.setFatal();

diff --git a/zoie-core/src/test/java/proj/zoie/test/ZoieTest.java b/zoie-core/src/test/java/proj/zoie/test/ZoieTest.java
@@ -26,7 +26,9 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queryParser.ParseException;
 import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.DocIdSet;
 import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
@@ -259,13 +261,63 @@ public void testRealtime2() throws ZoieException {
 		}
 	}
 
+	private static class EvenIDPurgeFilter extends Filter{
+
+    @Override
+    public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
+      if (reader instanceof ZoieIndexReader){
+        final ZoieIndexReader<IndexReader> zoieReader = (ZoieIndexReader<IndexReader>)reader;
+        return new DocIdSet(){
+
+          @Override
+          public DocIdSetIterator iterator() throws IOException {
+            return new DocIdSetIterator(){
+
+              int doc=-1;
+              int maxdoc = zoieReader.maxDoc();
+
+              @Override
+              public int advance(int target) throws IOException {
+                doc = target-1;
+                return nextDoc();
+              }
+
+              @Override
+              public int docID() {
+                return doc;
+              }
+
+              @Override
+              public int nextDoc() throws IOException {
+                while(doc<maxdoc){
+                  doc++;
+                  long uid = zoieReader.getUID(doc);
+                  if (uid %2 == 0){ // if even
+                    return doc;
+                  }
+                }
+                return DocIdSetIterator.NO_MORE_DOCS;
+              }
+
+            };
+          }
+
+        };
+      }
+      else{
+        throw new IllegalStateException("expecting instance of ZoieIndexReader, but got: "+reader.getClass());
+      }
+    }
+
+	}
+
 	@Test
   public void testPurgeFilter() throws Exception {
     File idxDir = getIdxDir();
     ZoieSystem<IndexReader, String> idxSystem = createZoie(
         idxDir, true, ZoieConfig.DEFAULT_VERSION_COMPARATOR,true);
 
-    idxSystem.setPurgeFilter(new QueryWrapperFilter(new MatchAllDocsQuery()));
+    idxSystem.setPurgeFilter(new EvenIDPurgeFilter());
     idxSystem.start();
 
     MemoryStreamDataProvider<String> memoryProvider = new MemoryStreamDataProvider<String>(ZoieConfig.DEFAULT_VERSION_COMPARATOR);
@@ -321,7 +373,7 @@ public void testPurgeFilter() throws Exception {
       numDocs = multiReader.numDocs();
 
       log.info("new numdocs: "+numDocs);
-      TestCase.assertTrue(numDocs==0);
+      TestCase.assertTrue(numDocs==5);
 
       idxSystem.returnIndexReaders(readers);
 

diff --git a/zoie-perf/src/main/java/proj/zoie/perf/indexing/LinedFileDataProvider.java b/zoie-perf/src/main/java/proj/zoie/perf/indexing/LinedFileDataProvider.java
@@ -27,7 +27,7 @@ public class LinedFileDataProvider extends StreamDataProvider<String> {
 
 
 	public LinedFileDataProvider(File file,long startingOffset){
-      super(ZoieConfig.DEFAULT_VERSION_COMPARATOR);
+      super(ZoiePerfVersion.COMPARATOR);
 	  _file = file;
 	  _rad = null;
 	  _startingOffset = startingOffset;
@@ -43,15 +43,15 @@ public DataEvent<String> next() {
 			if (line == null) return null;
 
 			String version = ZoiePerfVersion.toString(_count,_offset);
-			_offset+=version.length();
+			_offset+=line.length();
+	    _count++;
 
 			event = new DataEvent<String>(line,version);
 		  }
 		  catch(IOException ioe){
 			logger.error(ioe.getMessage(),ioe);
 		  }
 		}
-		_count++;
 		return event;
 	}