Permalink
Browse files

Merged from John's master.

  • Loading branch information...
2 parents 1d6a0dc + f819869 commit 0fbab2ce4870e4e3472f6515806483e468aa129b @wonlay wonlay committed Jun 30, 2011
View
@@ -25,7 +25,7 @@
<parent>
<groupId>com.linkedin.zoie</groupId>
<artifactId>zoie-parent</artifactId>
- <version>2.6.0-SNAPSHOT</version>
+ <version>2.6.0</version>
<relativePath>zoie-parent/pom.xml</relativePath>
</parent>
View
@@ -5,7 +5,7 @@
<parent>
<groupId>com.linkedin.zoie</groupId>
<artifactId>zoie-parent</artifactId>
- <version>2.6.0-SNAPSHOT</version>
+ <version>2.6.0</version>
<relativePath>../zoie-parent/pom.xml</relativePath>
</parent>
@@ -64,7 +64,7 @@
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
- <version>3.0.2</version>
+ <version>2.9.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
@@ -40,7 +40,7 @@ public static String getSegmentsInfo(Directory idx)
String x="";
for(int i=0;i<infos.size(); i++)
{
- x += "[numDoc:" + infos.info(i).docCount + ",numDel:"+infos.info(i).getDelCount()+"]";
+ x += "[" + infos.info(i).name + ",numDoc:" + infos.info(i).docCount + ",numDel:" + infos.info(i).getDelCount() + "]";
}
return x;
} catch (Exception e)
@@ -96,7 +96,7 @@ public void run()
hourglass._shutdownLock.readLock().unlock();
}
}
- }},"Zoie Maintenanace Thread");
+ }},"HourglassReaderManager Zoie Maintenanace Thread");
maintenanceThread.start();
}
/**
@@ -117,7 +117,7 @@ private Thread newMaintenanceThread()
{
public MaintenanceThread()
{
- super("zoie-indexReader-maintenance");
+ super("DefaultReaderCache-zoie-indexReader-maintenance");
}
@Override
public void run()
@@ -153,7 +153,7 @@ public void run()
}
} catch (IOException e)
{
- log.info("zoie-indexReader-maintenance", e);
+ log.info("DefaultReaderCache-zoie-indexReader-maintenance", e);
newreaders = new ArrayList<ZoieIndexReader<R>>();
}
}
@@ -96,12 +96,17 @@ public long getFreshness()
@Override
public void setFreshness(long freshness)
{
+ if (freshness < 0){
+ log.warn("freshness has to be at least 0 and cannot be " + freshness);
+ freshness = 10000;
+ }
+ log.info("setting freshness to " + freshness + "ms");
_freshness = freshness;
}
private Thread newMaintenanceThread()
{
- return new Thread("zoie-indexReader-maintenance")
+ return new Thread("SmartReaderCache-zoie-indexReader-maintenance")
{
@Override
public void run()
@@ -121,7 +126,7 @@ public void run()
List<ZoieIndexReader<R>> newreaders = null;
if (alreadyShutdown)
{
- newreaders = new ArrayList<ZoieIndexReader<R>>();
+ newreaders = new ArrayList<ZoieIndexReader<R>>(0);
// clean up and quit
} else
{
@@ -130,25 +135,31 @@ public void run()
newreaders = _readerfactory.getIndexReaders();
} catch (IOException e)
{
- log.info("zoie-indexReader-maintenance", e);
+ log.info("SmartReaderCache-zoie-indexReader-maintenance", e);
newreaders = new ArrayList<ZoieIndexReader<R>>();
}
}
cachedreaders = new ArrayList<ZoieIndexReader<R>>(newreaders);
- WeakReference<List<ZoieIndexReader<R>>> w = new WeakReference<List<ZoieIndexReader<R>>>(cachedreaders, refq);
- readermap.put(w, newreaders); // when nobody uses cachedreaders, we will clean newreaders :)
+ if (cachedreaders.size()>0){
+ WeakReference<List<ZoieIndexReader<R>>> w = new WeakReference<List<ZoieIndexReader<R>>>(cachedreaders, refq);
+ readermap.put(w, newreaders); // when nobody uses cachedreaders, we will clean newreaders :)
+ }
cachedreaderTimestamp = System.currentTimeMillis();
synchronized (cachemonitor)
{
cachemonitor.notifyAll();
}
- // clearning and reference counting on the ones no longer in use
+ // cleaning and reference counting on the ones no longer in use
Reference<? extends List<ZoieIndexReader<R>>> wclean = null;
while((wclean = refq.poll()) != null)
{
List<ZoieIndexReader<R>> readers = readermap.remove(wclean);
_readerfactory.returnIndexReaders(readers);
}
+ if (alreadyShutdown && readermap.size() == 0){
+ log.info("SmartReaderCache-zoie-indexReader-maintenance done.");
+ break;
+ }
}
}
};
@@ -35,6 +35,7 @@
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.DefaultSimilarity;
+import org.apache.lucene.search.Filter;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.util.Version;
@@ -438,6 +439,10 @@ public ZoieSystem(DirectoryManager dirMgr,
{
return new ZoieSystem<IndexReader, D>(idxDir, interpreter, new DefaultIndexReaderDecorator(), analyzer, similarity, batchSize, batchDelay, realtime, versionComparator);
}
+
+ public void setPurgeFilter(Filter purgeFilter){
+ _diskLoader.setPurgeFilter(purgeFilter);
+ }
public void addIndexingEventListener(IndexingEventListener lsnr)
{
@@ -31,6 +31,8 @@
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Filter;
import org.apache.lucene.search.Similarity;
import proj.zoie.api.DataConsumer;
@@ -47,18 +49,58 @@
protected final Similarity _similarity;
protected final SearchIndexManager<R> _idxMgr;
protected final Comparator<String> _versionComparator;
+ private Filter _purgeFilter;
protected LuceneIndexDataLoader(Analyzer analyzer, Similarity similarity,SearchIndexManager<R> idxMgr,Comparator<String> versionComparator) {
_analyzer = analyzer;
_similarity = similarity;
_idxMgr=idxMgr;
_versionComparator = versionComparator;
+ _purgeFilter = null;
+ }
+
+ public void setPurgeFilter(Filter purgeFilter){
+ _purgeFilter = purgeFilter;
}
protected abstract BaseSearchIndex<R> getSearchIndex();
protected abstract void propagateDeletes(LongSet delDocs) throws IOException;
protected abstract void commitPropagatedDeletes() throws IOException;
+
+ private final void purgeDocuments() throws IOException{
+ if (_purgeFilter!=null){
+ BaseSearchIndex<R> idx = getSearchIndex();
+ IndexReader reader = null;
+ log.info("purging docs started...");
+ int count = 0;
+ long start = System.currentTimeMillis();
+ try{
+ reader = idx.openIndexReaderForDelete();
+ DocIdSetIterator iter = _purgeFilter.getDocIdSet(reader).iterator();
+ int doc;
+ while((doc = iter.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS){
+ count++;
+ reader.deleteDocument(doc);
+ }
+ }
+ finally{
+ if (reader!=null){
+ try{
+ reader.close();
+ }
+ catch(IOException ioe){
+ ZoieHealth.setFatal();
+ log.error(ioe.getMessage(),ioe);
+ }
+ }
+ }
+
+ long end = System.currentTimeMillis();
+ log.info("purging docs completed in "+(end-start)+"ms");
+ log.info("total docs purged: " +count);
+ }
+ }
/**
* @Precondition incoming events sorted by version number
@@ -128,6 +170,7 @@ public void consume(Collection<DataEvent<ZoieIndexable>> events) throws ZoieExce
docList.addAll(tmpList);
}
idx.updateIndex(delSet, docList, _analyzer,_similarity);
+ purgeDocuments();
propagateDeletes(delSet);
synchronized(_idxMgr)
{
@@ -8,6 +8,8 @@
import javax.management.MBeanServer;
import javax.management.ObjectName;
+import org.apache.log4j.Appender;
+import org.apache.log4j.PatternLayout;
import org.apache.log4j.Logger;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.IndexReader;
@@ -25,6 +27,18 @@
public class ZoieTestCaseBase
{
+ static{
+ Appender appender = null;
+ PatternLayout layout = new PatternLayout();
+ layout.setConversionPattern("%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%x] %m%n");
+ try{
+ appender = new org.apache.log4j.FileAppender(layout,"zoie-test-log.log");
+ }
+ catch(IOException e){
+ e.printStackTrace();
+ }
+ org.apache.log4j.BasicConfigurator.configure(appender);
+ }
static Logger log = Logger.getLogger(ZoieTestCaseBase.class);
static MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
public void registerMBean(Object standardmbean, String mbeanname)
View
@@ -25,7 +25,7 @@
<parent>
<groupId>com.linkedin.zoie</groupId>
<artifactId>zoie-parent</artifactId>
- <version>2.6.0-SNAPSHOT</version>
+ <version>2.6.0</version>
<relativePath>../zoie-parent/pom.xml</relativePath>
</parent>
@@ -112,7 +112,7 @@
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-highlighter</artifactId>
- <version>3.0.2</version>
+ <version>2.9.1</version>
</dependency>
<dependency>
@@ -8,7 +8,7 @@
<bean id="fileInterpreter" class="proj.zoie.impl.indexing.FileIndexableInterpreter" />
- <bean id="versionComparator" class="proj.zoie.impl.indexing.ZoieConfig.DefaultVersionComparator" />
+ <bean id="versionComparator" class="proj.zoie.impl.indexing.ZoieConfig$DefaultVersionComparator" />
<bean id="indexingSystem" class="proj.zoie.impl.indexing.SimpleZoieSystem" init-method="start" destroy-method="shutdown">
<constructor-arg value="file:${index.dir}"/>
View
@@ -5,7 +5,7 @@
<parent>
<groupId>com.linkedin.zoie</groupId>
<artifactId>zoie-parent</artifactId>
- <version>2.6.0-SNAPSHOT</version>
+ <version>2.6.0</version>
<relativePath>../zoie-parent/pom.xml</relativePath>
</parent>
@@ -54,7 +54,7 @@
<repository>
<id>jbossReleases</id>
<name>Jboss Released Java Packages</name>
- <url>http://repository.jboss.com/maven2/</url>
+ <url>http://repository.jboss.org/nexus/content/groups/public/</url>
<layout>default</layout>
</repository>
</repositories>
View
@@ -17,7 +17,7 @@
<groupId>com.linkedin.zoie</groupId>
<artifactId>zoie-parent</artifactId>
- <version>2.6.0-SNAPSHOT</version>
+ <version>2.6.0</version>
<packaging>pom</packaging>
<name>zoie parent</name>
@@ -69,6 +69,14 @@
<role>committer</role>
</roles>
</developer>
+ <developer>
+ <name>Lei Wang</name>
+ <id>wonlay</id>
+ <email>wonlay@gmail.com</email>
+ <roles>
+ <role>committer</role>
+ </roles>
+ </developer>
</developers>
<dependencyManagement>
View
@@ -25,7 +25,7 @@
<parent>
<groupId>com.linkedin.zoie</groupId>
<artifactId>zoie-parent</artifactId>
- <version>2.6.0-SNAPSHOT</version>
+ <version>2.6.0</version>
<relativePath>../zoie-parent/pom.xml</relativePath>
</parent>
View
@@ -5,7 +5,7 @@
<parent>
<groupId>com.linkedin.zoie</groupId>
<artifactId>zoie-parent</artifactId>
- <version>2.6.0-SNAPSHOT</version>
+ <version>2.6.0</version>
<relativePath>../zoie-parent/pom.xml</relativePath>
</parent>
View
@@ -5,7 +5,7 @@
<parent>
<groupId>com.linkedin.zoie</groupId>
<artifactId>zoie-parent</artifactId>
- <version>2.6.0-SNAPSHOT</version>
+ <version>2.6.0</version>
<relativePath>../zoie-parent/pom.xml</relativePath>
</parent>
@@ -51,7 +51,7 @@ private ZoieSystemHome(SolrCore core){
}
catch(Exception e){
log.error(e.getMessage()+", defaulting to "+StandardAnalyzer.class,e);
- analyzer = new StandardAnalyzer(Version.LUCENE_30);
+ analyzer = new StandardAnalyzer(Version.LUCENE_29);
}
Similarity similarity = null;

0 comments on commit 0fbab2c

Please sign in to comment.