Permalink
Browse files

Merge branch 'master' of git://github.com/xiaoyang/bobo

  • Loading branch information...
ymatsuda committed May 5, 2010
2 parents 69ce9f3 + 94b1977 commit a6e8b2fac2cad091f084bd96d13256ca30465d47
@@ -67,6 +67,7 @@ public void load(String fieldName, IndexReader reader, TermListFactory<T> listFa
*/
public void load(String fieldName, IndexReader reader, TermListFactory<T> listFactory, WorkArea workArea) throws IOException
{
+ long t0 = System.currentTimeMillis();
int maxdoc = reader.maxDoc();
BufferedLoader loader = getBufferedLoader(maxdoc, workArea);
@@ -170,6 +171,7 @@ public void load(String fieldName, IndexReader reader, TermListFactory<T> listFa
this.freqs = freqList.toIntArray();
this.minIDs = minIDList.toIntArray();
this.maxIDs = maxIDList.toIntArray();
+ logger.info("loaded " + fieldName + " of type " + listFactory.getType() + " from " + reader +" in " + (System.currentTimeMillis()-t0) + " ms");
}
/**
@@ -5,6 +5,7 @@
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.log4j.Logger;
@@ -35,6 +36,7 @@
private int _docBase;
protected LinkedList<int[]> intarraylist = new LinkedList<int[]>();
private Iterator _iterator;
+ public static final AtomicLong al = new AtomicLong();
protected static MemoryManager<int[]> intarraymgr = new MemoryManager<int[]>(new MemoryManager.Initializer<int[]>()
{
@@ -64,8 +66,17 @@ public DefaultFacetCountCollector(String name,FacetDataCache dataCache,int docBa
_ospec = ospec;
_name = name;
_dataCache=dataCache;
- _count = intarraymgr.get(_dataCache.freqs.length);//new int[_dataCache.freqs.length];
- intarraylist.add(_count);
+ long t0 = System.nanoTime();
+ if (_dataCache.freqs.length < 512)
+ {
+ _count = new int[_dataCache.freqs.length];
+ } else
+ {
+ _count = intarraymgr.get(_dataCache.freqs.length);//new int[_dataCache.freqs.length];
+ intarraylist.add(_count);
+ }
+ long t1 = System.nanoTime();
+ al.addAndGet(t1-t0);
_array = _dataCache.orderArray;
_docBase = docBase;
}
@@ -19,7 +19,8 @@
private static final Logger log = Logger.getLogger(MemoryManager.class.getName());
private final ConcurrentHashMap<Integer, ConcurrentLinkedQueue<WeakReference<T>>> _sizeMap = new ConcurrentHashMap<Integer, ConcurrentLinkedQueue<WeakReference<T>>>();
- private final ConcurrentLinkedQueue<T> _releaseQueue = new ConcurrentLinkedQueue<T>();
+ private volatile ConcurrentLinkedQueue<T> _releaseQueue = new ConcurrentLinkedQueue<T>();
+ private volatile ConcurrentLinkedQueue<T> _releaseQueueb = new ConcurrentLinkedQueue<T>();
private final AtomicInteger _releaseQueueSize = new AtomicInteger(0);
private final Initializer<T> _initializer;
private final Thread _cleanThread;
@@ -37,19 +38,22 @@ public void run()
{
try
{
- MemoryManager.this.wait(200);
+ MemoryManager.this.wait(10);
} catch (InterruptedException e)
{
log.error(e);
}
}
- while((buf = _releaseQueue.poll()) != null)
+ ConcurrentLinkedQueue<T> t = _releaseQueue;
+ _releaseQueue = _releaseQueueb;
+ _releaseQueueb =t;
+ while((buf = _releaseQueueb.poll()) != null)
{
ConcurrentLinkedQueue<WeakReference<T>> queue = _sizeMap.get(_initializer.size(buf));
// buf is wrapped in WeakReference. this allows GC to reclaim the buffer memory
_initializer.init(buf);// pre-initializing the buffer in parallel so we save time when it is requested later.
queue.offer(new WeakReference<T>(buf));
- _releaseQueueSize.decrementAndGet();
+ int x =_releaseQueueSize.decrementAndGet();
}
buf = null;
}
@@ -94,9 +98,13 @@ public T get(int size)
*/
public void release(T buf)
{
- if (_releaseQueueSize.get()>1000)
+ if (_releaseQueueSize.get()>8000)
{
log.info("release queue full");
+ synchronized(MemoryManager.this)
+ {
+ MemoryManager.this.notifyAll();
+ }
return;
}
if(buf != null)
View
@@ -294,4 +294,25 @@
<report todir="${test.html.dir}" format="frames" />
</junitreport>
</target>
-</project>
+ <target name="lucene-test" description="Lucene-test" depends="test-build">
+ <echo>=== Running Lucene 2.9 Perf ===</echo>
+ <java fork="true" classname="com.browseengine.bobo.test.LucenePerfTest">
+ <jvmarg value="-d64" />
+ <jvmarg value="-server" />
+ <jvmarg value="-Xms8g" />
+ <jvmarg value="-Xmx10g" />
+<jvmarg value="-XX:+UseConcMarkSweepGC"/>
+<jvmarg value="-XX:+UseParNewGC"/>
+<jvmarg value="-XX:CMSMarkStackSize=32m"/>
+<jvmarg value="-XX:MaxTenuringThreshold=2"/>
+<jvmarg value="-XX:CMSInitiatingOccupancyFraction=70"/>
+<jvmarg value="-XX:+PrintTenuringDistribution"/>
+<jvmarg value="-XX:+PrintGCDetails"/>
+<jvmarg value="-XX:+PrintGCTimeStamps"/>
+<jvmarg value="-Xloggc:gc.log"/>
+<jvmarg value="-XX:NewSize=1536m"/>
+<jvmarg value="-XX:MaxNewSize=1536m"/>
+<classpath refid="test.compile.class.path"/>
+ </java>
+ </target>
+</project>
@@ -1,32 +1,30 @@
package com.browseengine.bobo.test;
import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.LineNumberReader;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
-import java.util.List;
import java.util.Random;
import java.util.Map.Entry;
+import org.apache.lucene.LucenePackage;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.SimpleFSDirectory;
import com.browseengine.bobo.api.BoboBrowser;
import com.browseengine.bobo.api.BoboIndexReader;
import com.browseengine.bobo.api.BrowseException;
-import com.browseengine.bobo.api.BrowseFacet;
import com.browseengine.bobo.api.BrowseRequest;
import com.browseengine.bobo.api.BrowseResult;
import com.browseengine.bobo.api.BrowseSelection;
@@ -35,9 +33,9 @@
import com.browseengine.bobo.api.FacetSpec.FacetSortSpec;
import com.browseengine.bobo.facets.FacetHandler;
import com.browseengine.bobo.facets.data.PredefinedTermListFactory;
+import com.browseengine.bobo.facets.impl.DefaultFacetCountCollector;
import com.browseengine.bobo.facets.impl.MultiValueFacetHandler;
import com.browseengine.bobo.facets.impl.SimpleFacetHandler;
-import com.browseengine.bobo.search.BoboSearcher2;
public class LucenePerfTest
{
@@ -46,35 +44,75 @@
"services", "on", "senior", "marketing", "project", "sales",
"technology", "systems", "as", "software", "new", "professional",
"owner", "experience", "inc", "team", "company" };
+ public static final ArrayList<String> wordlist = new ArrayList<String>();
+ static final Random rand = new Random(987129);
+ public static int inNumItr = 10;
/**
* @param args
* @throws IOException
* @throws CorruptIndexException
* @throws InterruptedException
* @throws BrowseException
*/
- public static void main(String[] args) throws CorruptIndexException,
- IOException, InterruptedException, BrowseException
+ public static void main(String[] args) throws CorruptIndexException,IOException, InterruptedException, BrowseException
{
- int numThread = 40;
- Thread[] threads = new Thread[numThread];
+ System.out.println(LucenePackage.get());
File file = new File("/Users/xgu/lucene29test/caches/people-search-index");
-// FSDirectory directory = new SimpleFSDirectory(file);
- FSDirectory directory = FSDirectory.getDirectory(file);
+ FSDirectory directory = new SimpleFSDirectory(file);
+// FSDirectory directory = FSDirectory.getDirectory(file);
+ System.out.println(directory.getClass().getName());
IndexReader reader = IndexReader.open(directory, true);
- final Random rand = new Random(987129);
+ loadFile();
+// TermEnum termEnum = reader.terms(new Term("b", ""));
+// while(termEnum.next())
+// {
+// Term t = termEnum.term();
+// wordlist.add(t.text());
+// }
+// words = wordlist.toArray(new String[1]);
+ System.out.println("load the words " + words.length);
+
final Collection<FacetHandler<?>> facetHandlers = new ArrayList<FacetHandler<?>>();
facetHandlers.add(new MultiValueFacetHandler("ccid", new PredefinedTermListFactory<Integer>(Integer.class,"0000000000")));
facetHandlers.add(new SimpleFacetHandler("industry", new PredefinedTermListFactory<Integer>(Integer.class,"0000000000")));
+ facetHandlers.add(new MultiValueFacetHandler("education_id", new PredefinedTermListFactory<Integer>(Integer.class,"0000000000")));
final BoboIndexReader boboReader = BoboIndexReader.getInstance(reader, facetHandlers , null);
+//warming
+ for(int x=0; x<30; x++)
+ {
+ doSearch(5, boboReader, facetHandlers);
+ }
+long initvalue = DefaultFacetCountCollector.al.get();
+ long start = System.currentTimeMillis();
+ int numThread = 1;
+ System.out.println(numThread+" threads");
+ int numItr = 200;
+ long ttime = 0;
+ for(int x=0; x<numItr; x++)
+ {
+ long time = doSearch(numThread, boboReader, facetHandlers);
+ ttime += time;
+ }
+ System.out.println("\n\nallocation: "+ (DefaultFacetCountCollector.al.get() -initvalue)/1000000);
+ System.out.println("total time: " + ttime);
+ System.out.println("number of iterations: "+ numItr + "\t\tnumThread: "+ numThread + "\t\tinner itr: " +inNumItr);
+ System.out.println("average time: " + (ttime/numItr/numThread/inNumItr));
+ System.out.println(LucenePackage.get());
+ }
+ private static long doSearch(int numThread,final BoboIndexReader boboReader, final Collection<FacetHandler<?>> facetHandlers) throws IOException,
+ CorruptIndexException, InterruptedException
+ {
+ Thread[] threads = new Thread[numThread];
+ final long[] times = new long[numThread];
for(int x =0; x<threads.length; x++)
{
+ final int y = x;
threads[x] = new Thread()
{public void run(){
try
{
- oneRun(boboReader, facetHandlers);
+ times[y] = oneRun(boboReader, facetHandlers);
} catch (IOException e)
{
// TODO Auto-generated catch block
@@ -91,11 +129,13 @@ public static void main(String[] args) throws CorruptIndexException,
threads[x].setDaemon(true);
threads[x].start();
}
+ long sum = 0;
for(int x =0; x<threads.length; x++)
{
threads[x].join();
+ sum+= times[x];
}
- System.out.println("2.9");
+ return sum;
}
/*
* [00000001371(3156), 00000001025(2951), 00000001035(2688), 00000001009(2429), 00000157234(2318), 00000001028(1871),
@@ -109,23 +149,25 @@ public static void main(String[] args) throws CorruptIndexException,
* 00000001482(382), -00000000001(0)]
*/
- private static void oneRun(BoboIndexReader boboReader,
+ private static long oneRun(BoboIndexReader boboReader,
Collection<FacetHandler<?>> facetHandlers) throws IOException,
BrowseException
{
- int numItr = 15;
long tt = 0;
- for(int x=0; x< numItr; x++)
+ long hitscount = 0;
+ for(int x=0; x< inNumItr; x++)
{
long t0 = System.currentTimeMillis();
BoboBrowser browser = new BoboBrowser(boboReader);
BrowseRequest req = new BrowseRequest();
+ req.setCount(50);
FacetSpec spec = new FacetSpec();
spec.setMaxCount(50);
spec.setOrderBy(FacetSortSpec.OrderHitsDesc);
-// req.setFacetSpec("ccid", spec);
-// req.setFacetSpec("industry", spec);
- req.setQuery(new TermQuery(new Term("b","company")));
+ req.setFacetSpec("ccid", spec);
+ req.setFacetSpec("education_id", spec);
+ req.setFacetSpec("industry", spec);
+ req.setQuery(new TermQuery(new Term("b",words[rand.nextInt(words.length)])));
BrowseSelection sel = new BrowseSelection("ccid");
sel.addValue("0000001384");
// req.addSelection(sel );
@@ -140,18 +182,49 @@ private static void oneRun(BoboIndexReader boboReader,
// System.out.println(entry.getKey());
FacetAccessible fa = entry.getValue();
tf0 = System.currentTimeMillis();
- List<BrowseFacet> facets = fa.getFacets();
+// List<BrowseFacet> facets = fa.getFacets();
tf1=System.currentTimeMillis();
// System.out.println(tf1 - tf0 + "\tfacet get time\tsize: " + facets.size());
// System.out.println(Arrays.toString(facets.toArray()));
+ fa.close();
}
- long t2 = System.currentTimeMillis();
// System.out.println(t2 - t0 +"\ttotal time\t\t\t hits: "+ bres.getNumHits());
+ hitscount += bres.getNumHits();
+ long t2 = System.currentTimeMillis();
tt+= (t2 - t0);
// System.out.println(t2 - t0 -(tf1-tf0)+"\tsearch time\t");
}
- System.out.println(tt/numItr);
+ System.out.println("hits count: " + hitscount);
+ try
+ {
+ Thread.sleep(50);
+ } catch (InterruptedException e)
+ {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ return tt;
+ }
+
+ public static void loadFile()
+ {
+ File file = new File("/Users/xgu/lucene29test/keywords");
+ try
+ {
+ FileInputStream fis = new FileInputStream("/Users/xgu/lucene29test/keywords");
+ InputStreamReader isr = new InputStreamReader(fis);
+ LineNumberReader reader = new LineNumberReader(isr);
+ String line;
+ while( (line=reader.readLine())!=null)
+ {
+ wordlist.add(line.split(" ")[0]);
+ }
+ words = wordlist.toArray(new String[1]);
+ } catch (Exception e)
+ {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
}
-
-
}

0 comments on commit a6e8b2f

Please sign in to comment.