Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

bug fix ZOIE-106

  • Loading branch information...
commit 094894032a38ffe848585d88c43c0ad6823f1539 1 parent 0270c70
authored May 28, 2012
37  zoie-core/src/main/java/proj/zoie/api/DataDoc.java
... ...
@@ -0,0 +1,37 @@
  1
+package proj.zoie.api;
  2
+
  3
+import org.apache.lucene.document.Document;
  4
+
  5
+import proj.zoie.api.indexing.AbstractZoieIndexable;
  6
+
  7
+public class DataDoc extends AbstractZoieIndexable {
  8
+	private long uid;
  9
+	private Document doc;
  10
+	private boolean valid;
  11
+	public DataDoc(long uid, Document doc) {
  12
+		this.uid = uid;
  13
+		this.doc = doc;
  14
+		this.valid = true;
  15
+	}
  16
+	
  17
+	public DataDoc(long uid) {
  18
+		this.uid = uid;
  19
+		this.valid = false;
  20
+	}
  21
+	
  22
+	@Override
  23
+	public IndexingReq[] buildIndexingReqs() {
  24
+		return new IndexingReq[]{new IndexingReq(doc)};
  25
+	}
  26
+
  27
+	@Override
  28
+	public long getUID() {
  29
+		return uid;
  30
+	}
  31
+
  32
+	@Override
  33
+	public boolean isDeleted() {
  34
+		return !valid;
  35
+	}
  36
+
  37
+}
34  zoie-core/src/main/java/proj/zoie/impl/indexing/internal/ZoieSegmentTermDocs.java
@@ -2,23 +2,43 @@
2 2
 
3 3
 import java.io.IOException;
4 4
 
5  
-import org.apache.lucene.index.TermDocs;
6 5
 import org.apache.lucene.index.FilterIndexReader.FilterTermDocs;
  6
+import org.apache.lucene.index.Term;
  7
+import org.apache.lucene.index.TermDocs;
  8
+import org.apache.lucene.index.TermEnum;
7 9
 import org.apache.lucene.search.DocIdSet;
8 10
 import org.apache.lucene.search.DocIdSetIterator;
9 11
 
10 12
 public class ZoieSegmentTermDocs extends FilterTermDocs {
11  
-	private final DocIdSetIterator _delSetIterator;
  13
+	private final DocIdSet delSet;
  14
+	private int _firstDelDoc = -1;
  15
+	private DocIdSetIterator _delSetIterator;
12 16
     private int _nextDelDoc;
13 17
     
14 18
 	public ZoieSegmentTermDocs(TermDocs in,DocIdSet delSet) throws IOException{
15  
-		this(in,delSet.iterator());
  19
+		super(in);
  20
+		this.delSet = delSet;
  21
+		resetDelIter();
  22
+		_firstDelDoc = _nextDelDoc;
16 23
 	}
17 24
 	
18  
-	public ZoieSegmentTermDocs(TermDocs in, DocIdSetIterator delSetIterator) throws IOException {
19  
-		super(in);
20  
-		_delSetIterator = delSetIterator;
21  
-		_nextDelDoc=_delSetIterator.nextDoc();
  25
+	private final void resetDelIter() throws IOException {
  26
+		if (_firstDelDoc != _nextDelDoc) {
  27
+			_delSetIterator = delSet.iterator();
  28
+			_nextDelDoc = _delSetIterator.nextDoc();
  29
+		}
  30
+	}
  31
+	
  32
+	@Override
  33
+	public void seek(Term term) throws IOException {
  34
+		resetDelIter();
  35
+		super.seek(term);
  36
+	}
  37
+
  38
+	@Override
  39
+	public void seek(TermEnum termEnum) throws IOException {
  40
+		resetDelIter();
  41
+		super.seek(termEnum);
22 42
 	}
23 43
 
24 44
 	public boolean next() throws IOException {
10  zoie-core/src/main/java/proj/zoie/impl/indexing/internal/ZoieSegmentTermPositions.java
@@ -4,21 +4,15 @@
4 4
 
5 5
 import org.apache.lucene.index.TermPositions;
6 6
 import org.apache.lucene.search.DocIdSet;
7  
-import org.apache.lucene.search.DocIdSetIterator;
8 7
 
9 8
 public class ZoieSegmentTermPositions extends ZoieSegmentTermDocs implements TermPositions
10 9
 {
11 10
 	  final TermPositions _tp;
12 11
 	  
13  
-	  public ZoieSegmentTermPositions(TermPositions in, DocIdSetIterator delSetIterator) throws IOException
14  
-	  {
15  
-	    super(in, delSetIterator);
16  
-	    _tp = in;
17  
-	  }
18  
-	  
19 12
 	  public ZoieSegmentTermPositions(TermPositions in, DocIdSet delSet) throws IOException
20 13
 	  {
21  
-	    this(in,delSet.iterator());
  14
+	    super(in,delSet);
  15
+	    _tp=in;
22 16
 	  }
23 17
 	  
24 18
 	  public int nextPosition() throws IOException
94  zoie-core/src/test/java/proj/zoie/test/ZoieTest.java
@@ -19,7 +19,12 @@
19 19
 
20 20
 import org.apache.log4j.Logger;
21 21
 import org.apache.lucene.analysis.WhitespaceAnalyzer;
  22
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
22 23
 import org.apache.lucene.document.Document;
  24
+import org.apache.lucene.document.Field;
  25
+import org.apache.lucene.document.Field.Index;
  26
+import org.apache.lucene.document.Field.Store;
  27
+import org.apache.lucene.document.Fieldable;
23 28
 import org.apache.lucene.index.CorruptIndexException;
24 29
 import org.apache.lucene.index.IndexReader;
25 30
 import org.apache.lucene.index.MultiReader;
@@ -41,6 +46,7 @@
41 46
 import org.junit.Test;
42 47
 
43 48
 import proj.zoie.api.DataConsumer.DataEvent;
  49
+import proj.zoie.api.DataDoc;
44 50
 import proj.zoie.api.DefaultDirectoryManager;
45 51
 import proj.zoie.api.DirectoryManager;
46 52
 import proj.zoie.api.DocIDMapper;
@@ -51,6 +57,8 @@
51 57
 import proj.zoie.api.impl.DocIDMapperImpl;
52 58
 import proj.zoie.api.impl.InRangeDocIDMapperFactory;
53 59
 import proj.zoie.api.indexing.IndexingEventListener;
  60
+import proj.zoie.api.indexing.ZoieIndexable;
  61
+import proj.zoie.api.indexing.ZoieIndexableInterpreter;
54 62
 import proj.zoie.impl.indexing.AsyncDataConsumer;
55 63
 import proj.zoie.impl.indexing.MemoryStreamDataProvider;
56 64
 import proj.zoie.impl.indexing.ZoieConfig;
@@ -370,6 +378,88 @@ public void handleIndexingEvent(IndexingEvent evt) {
370 378
     assertTrue(flushNum[0]>0);
371 379
     assertEquals("9", flushVersion[0]);
372 380
   }
  381
+  
  382
+  @Test
  383
+  public void testSegmentTermDocs() throws Exception{
  384
+	  
  385
+	  class DefaultInterpreter implements ZoieIndexableInterpreter<DataDoc> {
  386
+
  387
+			@Override
  388
+			public ZoieIndexable convertAndInterpret(DataDoc src) {
  389
+				return src;
  390
+			}
  391
+			
  392
+		}
  393
+	  
  394
+	  File idxDir = getIdxDir();
  395
+	  
  396
+	  ZoieConfig zConfig = new ZoieConfig();
  397
+		
  398
+      ZoieSystem<?, DataDoc> zoie = ZoieSystem.buildDefaultInstance(
  399
+				idxDir,
  400
+				new DefaultInterpreter(),
  401
+				zConfig);
  402
+	  zoie.start();
  403
+	    
  404
+	  Document d1 = new Document();
  405
+	  Fieldable f1 = new Field("num", "abcdef", Store.YES, Index.NOT_ANALYZED_NO_NORMS);
  406
+	  d1.add(f1);
  407
+		
  408
+	  Document d2 = new Document();
  409
+	  Fieldable f2 = new Field("num", "abcd", Store.YES, Index.NOT_ANALYZED_NO_NORMS);
  410
+	  d2.add(f2);
  411
+		
  412
+	  Document d3 = new Document();
  413
+	  Fieldable f3 = new Field("num", "abcde", Store.YES, Index.NOT_ANALYZED_NO_NORMS);
  414
+	  d3.add(f3);
  415
+		
  416
+		
  417
+	  DataEvent<DataDoc> de1 = new DataEvent<DataDoc>(new DataDoc(1, d1), "1");
  418
+	  DataEvent<DataDoc> de2 = new DataEvent<DataDoc>(new DataDoc(2, d2), "1");
  419
+	  DataEvent<DataDoc> de3 = new DataEvent<DataDoc>(new DataDoc(3, d3), "1");
  420
+		
  421
+	  try{
  422
+	    zoie.consume(Arrays.asList(de1, de2, de3));
  423
+	    zoie.flushEvents(10000);
  424
+	  
  425
+	    List<?> readerList = zoie.getIndexReaders();
  426
+	    // combine the readers
  427
+	    MultiReader reader = new MultiReader(readerList.toArray(new IndexReader[readerList.size()]),false);
  428
+	    // do search
  429
+	    IndexSearcher searcher = new IndexSearcher(reader);
  430
+	    QueryParser parser = new QueryParser(Version.LUCENE_35, "num", new StandardAnalyzer(Version.LUCENE_35));
  431
+	    Query q = parser.parse("num:abc*");
  432
+	    TopDocs ret = searcher.search(q, 100);
  433
+	    TestCase.assertEquals(3, ret.totalHits);
  434
+	    searcher.close();
  435
+	  
  436
+	    zoie.returnIndexReaders((List) readerList);
  437
+		
  438
+	    de1 = new DataEvent<DataDoc>(new DataDoc(1), "2");
  439
+	    de2 = new DataEvent<DataDoc>(new DataDoc(2), "2");
  440
+	    de3 = new DataEvent<DataDoc>(new DataDoc(3), "2");
  441
+	    zoie.consume(Arrays.asList(de1, de2, de3));
  442
+		
  443
+	    zoie.flushEventsToMemoryIndex(10000);
  444
+		
  445
+	    readerList = zoie.getIndexReaders();
  446
+	    // combine the readers
  447
+	     reader = new MultiReader(readerList.toArray(new IndexReader[readerList.size()]),false);
  448
+		// do search
  449
+	    searcher = new IndexSearcher(reader);
  450
+	    ret = searcher.search(q, 100);
  451
+	    searcher.close();
  452
+	    TestCase.assertEquals(0, ret.totalHits);
  453
+	    zoie.returnIndexReaders((List)readerList);
  454
+	  } 
  455
+	  catch (IOException ioe) {
  456
+	      throw new ZoieException(ioe.getMessage());
  457
+	  } 
  458
+	  finally {
  459
+	      zoie.shutdown();
  460
+	      deleteDirectory(idxDir);
  461
+	  }
  462
+  }
373 463
 
374 464
 	@Test
375 465
   public void testPurgeFilter() throws Exception {
@@ -1458,4 +1548,8 @@ public void testUIDDocIdSet() throws IOException {
1458 1548
 		assertTrue("wrong result from mix of next and skip",
1459 1549
 				Arrays.equals(answer, intList.toIntArray()));
1460 1550
 	}
  1551
+	
  1552
+	public static void main(String[] args) {
  1553
+		
  1554
+	}
1461 1555
 }

0 notes on commit 0948940

Please sign in to comment.
Something went wrong with that request. Please try again.