Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

bug fix ZOIE-106

  • Loading branch information...
commit 094894032a38ffe848585d88c43c0ad6823f1539 1 parent 0270c70
John Wang authored
View
37 zoie-core/src/main/java/proj/zoie/api/DataDoc.java
@@ -0,0 +1,37 @@
+package proj.zoie.api;
+
+import org.apache.lucene.document.Document;
+
+import proj.zoie.api.indexing.AbstractZoieIndexable;
+
+public class DataDoc extends AbstractZoieIndexable {
+ private long uid;
+ private Document doc;
+ private boolean valid;
+ public DataDoc(long uid, Document doc) {
+ this.uid = uid;
+ this.doc = doc;
+ this.valid = true;
+ }
+
+ public DataDoc(long uid) {
+ this.uid = uid;
+ this.valid = false;
+ }
+
+ @Override
+ public IndexingReq[] buildIndexingReqs() {
+ return new IndexingReq[]{new IndexingReq(doc)};
+ }
+
+ @Override
+ public long getUID() {
+ return uid;
+ }
+
+ @Override
+ public boolean isDeleted() {
+ return !valid;
+ }
+
+}
View
34 zoie-core/src/main/java/proj/zoie/impl/indexing/internal/ZoieSegmentTermDocs.java
@@ -2,23 +2,43 @@
import java.io.IOException;
-import org.apache.lucene.index.TermDocs;
import org.apache.lucene.index.FilterIndexReader.FilterTermDocs;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.TermEnum;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
public class ZoieSegmentTermDocs extends FilterTermDocs {
- private final DocIdSetIterator _delSetIterator;
+ private final DocIdSet delSet;
+ private int _firstDelDoc = -1;
+ private DocIdSetIterator _delSetIterator;
private int _nextDelDoc;
public ZoieSegmentTermDocs(TermDocs in,DocIdSet delSet) throws IOException{
- this(in,delSet.iterator());
+ super(in);
+ this.delSet = delSet;
+ resetDelIter();
+ _firstDelDoc = _nextDelDoc;
}
- public ZoieSegmentTermDocs(TermDocs in, DocIdSetIterator delSetIterator) throws IOException {
- super(in);
- _delSetIterator = delSetIterator;
- _nextDelDoc=_delSetIterator.nextDoc();
+ private final void resetDelIter() throws IOException {
+ if (_firstDelDoc != _nextDelDoc) {
+ _delSetIterator = delSet.iterator();
+ _nextDelDoc = _delSetIterator.nextDoc();
+ }
+ }
+
+ @Override
+ public void seek(Term term) throws IOException {
+ resetDelIter();
+ super.seek(term);
+ }
+
+ @Override
+ public void seek(TermEnum termEnum) throws IOException {
+ resetDelIter();
+ super.seek(termEnum);
}
public boolean next() throws IOException {
View
10 zoie-core/src/main/java/proj/zoie/impl/indexing/internal/ZoieSegmentTermPositions.java
@@ -4,21 +4,15 @@
import org.apache.lucene.index.TermPositions;
import org.apache.lucene.search.DocIdSet;
-import org.apache.lucene.search.DocIdSetIterator;
public class ZoieSegmentTermPositions extends ZoieSegmentTermDocs implements TermPositions
{
final TermPositions _tp;
- public ZoieSegmentTermPositions(TermPositions in, DocIdSetIterator delSetIterator) throws IOException
- {
- super(in, delSetIterator);
- _tp = in;
- }
-
public ZoieSegmentTermPositions(TermPositions in, DocIdSet delSet) throws IOException
{
- this(in,delSet.iterator());
+ super(in,delSet);
+ _tp=in;
}
public int nextPosition() throws IOException
View
94 zoie-core/src/test/java/proj/zoie/test/ZoieTest.java
@@ -19,7 +19,12 @@
import org.apache.log4j.Logger;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiReader;
@@ -41,6 +46,7 @@
import org.junit.Test;
import proj.zoie.api.DataConsumer.DataEvent;
+import proj.zoie.api.DataDoc;
import proj.zoie.api.DefaultDirectoryManager;
import proj.zoie.api.DirectoryManager;
import proj.zoie.api.DocIDMapper;
@@ -51,6 +57,8 @@
import proj.zoie.api.impl.DocIDMapperImpl;
import proj.zoie.api.impl.InRangeDocIDMapperFactory;
import proj.zoie.api.indexing.IndexingEventListener;
+import proj.zoie.api.indexing.ZoieIndexable;
+import proj.zoie.api.indexing.ZoieIndexableInterpreter;
import proj.zoie.impl.indexing.AsyncDataConsumer;
import proj.zoie.impl.indexing.MemoryStreamDataProvider;
import proj.zoie.impl.indexing.ZoieConfig;
@@ -370,6 +378,88 @@ public void handleIndexingEvent(IndexingEvent evt) {
assertTrue(flushNum[0]>0);
assertEquals("9", flushVersion[0]);
}
+
+ @Test
+ public void testSegmentTermDocs() throws Exception{
+
+ class DefaultInterpreter implements ZoieIndexableInterpreter<DataDoc> {
+
+ @Override
+ public ZoieIndexable convertAndInterpret(DataDoc src) {
+ return src;
+ }
+
+ }
+
+ File idxDir = getIdxDir();
+
+ ZoieConfig zConfig = new ZoieConfig();
+
+ ZoieSystem<?, DataDoc> zoie = ZoieSystem.buildDefaultInstance(
+ idxDir,
+ new DefaultInterpreter(),
+ zConfig);
+ zoie.start();
+
+ Document d1 = new Document();
+ Fieldable f1 = new Field("num", "abcdef", Store.YES, Index.NOT_ANALYZED_NO_NORMS);
+ d1.add(f1);
+
+ Document d2 = new Document();
+ Fieldable f2 = new Field("num", "abcd", Store.YES, Index.NOT_ANALYZED_NO_NORMS);
+ d2.add(f2);
+
+ Document d3 = new Document();
+ Fieldable f3 = new Field("num", "abcde", Store.YES, Index.NOT_ANALYZED_NO_NORMS);
+ d3.add(f3);
+
+
+ DataEvent<DataDoc> de1 = new DataEvent<DataDoc>(new DataDoc(1, d1), "1");
+ DataEvent<DataDoc> de2 = new DataEvent<DataDoc>(new DataDoc(2, d2), "1");
+ DataEvent<DataDoc> de3 = new DataEvent<DataDoc>(new DataDoc(3, d3), "1");
+
+ try{
+ zoie.consume(Arrays.asList(de1, de2, de3));
+ zoie.flushEvents(10000);
+
+ List<?> readerList = zoie.getIndexReaders();
+ // combine the readers
+ MultiReader reader = new MultiReader(readerList.toArray(new IndexReader[readerList.size()]),false);
+ // do search
+ IndexSearcher searcher = new IndexSearcher(reader);
+ QueryParser parser = new QueryParser(Version.LUCENE_35, "num", new StandardAnalyzer(Version.LUCENE_35));
+ Query q = parser.parse("num:abc*");
+ TopDocs ret = searcher.search(q, 100);
+ TestCase.assertEquals(3, ret.totalHits);
+ searcher.close();
+
+ zoie.returnIndexReaders((List) readerList);
+
+ de1 = new DataEvent<DataDoc>(new DataDoc(1), "2");
+ de2 = new DataEvent<DataDoc>(new DataDoc(2), "2");
+ de3 = new DataEvent<DataDoc>(new DataDoc(3), "2");
+ zoie.consume(Arrays.asList(de1, de2, de3));
+
+ zoie.flushEventsToMemoryIndex(10000);
+
+ readerList = zoie.getIndexReaders();
+ // combine the readers
+ reader = new MultiReader(readerList.toArray(new IndexReader[readerList.size()]),false);
+ // do search
+ searcher = new IndexSearcher(reader);
+ ret = searcher.search(q, 100);
+ searcher.close();
+ TestCase.assertEquals(0, ret.totalHits);
+ zoie.returnIndexReaders((List)readerList);
+ }
+ catch (IOException ioe) {
+ throw new ZoieException(ioe.getMessage());
+ }
+ finally {
+ zoie.shutdown();
+ deleteDirectory(idxDir);
+ }
+ }
@Test
public void testPurgeFilter() throws Exception {
@@ -1458,4 +1548,8 @@ public void testUIDDocIdSet() throws IOException {
assertTrue("wrong result from mix of next and skip",
Arrays.equals(answer, intList.toIntArray()));
}
+
+ public static void main(String[] args) {
+
+ }
}
Please sign in to comment.
Something went wrong with that request. Please try again.