@@ -48,7 +48,7 @@ <entry key="default" value-ref="itemSearch1"/> <entry key="aawaj" value-ref="itemSearch2"/> <entry key="spellchecker" value-ref="itemSearchSpellChecker"/> - <entry key="location" value-ref="itemSearchLocation"/> + <entry key="location" value-ref="itemSearchLocation"/> </map> </property> </bean> development/config/index-configuration/indexer-beans.xml @@ -24,6 +24,8 @@ import java.util.List; public class TermRequest { private List<String> mFields; private List<Integer> mItemIds; + private int mOffset; + private int mMax; public List<String> getFields() { return mFields; @@ -40,4 +42,20 @@ public class TermRequest { public void setItemIds(final List<Integer> pItemIds) { mItemIds = pItemIds; } + + public int getOffset() { + return mOffset; + } + + public void setOffset(final int pOffset) { + mOffset = pOffset; + } + + public int getMax() { + return mMax; + } + + public void setMax(final int pMax) { + mMax = pMax; + } } development/idea-repository-core/src/main/java/com/ideabase/repository/core/dao/TermRequest.java @@ -62,6 +62,8 @@ public class TermDAOImpl extends SqlMapClientDaoSupport implements TermDAO { final TermRequest request = new TermRequest(); request.setFields(pFields); request.setItemIds(pItemIds); + request.setMax(pMax); + request.setOffset(pOffset); return getSqlMapClientTemplate().queryForList( QUERY_FIND_TERMS_BY_ITEM_IDS, request, pOffset, pMax); } development/idea-repository-core/src/main/java/com/ideabase/repository/core/dao/impl/TermDAOImpl.java @@ -30,6 +30,7 @@ public class TermUsageAwareIndexFactoryBean extends SimpleIndexFactoryBean { private TermUsageAwareIndexFactory mFactory; private TermUsageService mTermUsageService; + private boolean mEnableAsyncTermStore = false; public void setTermUsageService(final TermUsageService pTermUsageService) { mTermUsageService = pTermUsageService; @@ -44,6 +45,15 @@ public class TermUsageAwareIndexFactoryBean extends SimpleIndexFactoryBean { super.afterPropertiesSet(); mFactory = new TermUsageAwareIndexFactory( - (SimpleIndexFactory) super.getObject(), mTermUsageService); + (SimpleIndexFactory) super.getObject(), mTermUsageService); + mFactory.setEnableAsyncTermStore(mEnableAsyncTermStore); + } + + public boolean isEnableAsyncTermStore() { + return mEnableAsyncTermStore; + } + + public void setEnableAsyncTermStore(final boolean pEnableAsyncTermStore) { + mEnableAsyncTermStore = pEnableAsyncTermStore; } } development/idea-repository-core/src/main/java/com/ideabase/repository/core/index/bean/TermUsageAwareIndexFactoryBean.java @@ -30,6 +30,7 @@ public class TermUsageAwareIndexFactory implements IndexFactory { private final SimpleIndexFactory mBaseIndexFactory; public static TermUsageService mTermUsageService; + private boolean mEnableAsyncTermStore = false; public TermUsageAwareIndexFactory( final SimpleIndexFactory pBaseIndexFactory, @@ -39,12 +40,26 @@ public class TermUsageAwareIndexFactory implements IndexFactory { } public LuceneIndexReader getIndexReader() { + return new TermUsageAwareLuceneIndexReader( - mBaseIndexFactory.getIndexReader(), mTermUsageService); + mBaseIndexFactory.getIndexReader(), + mTermUsageService, + mEnableAsyncTermStore); } public LuceneIndexWriter getIndexWriter() { + return new TermUsageAwareLuceneIndexWriter( - mBaseIndexFactory.getIndexWriter(), mTermUsageService); + mBaseIndexFactory.getIndexWriter(), + mTermUsageService, + mEnableAsyncTermStore); + } + + public boolean isEnableAsyncTermStore() { + return mEnableAsyncTermStore; + } + + public void setEnableAsyncTermStore(final boolean pEnableAsyncTermStore) { + mEnableAsyncTermStore = pEnableAsyncTermStore; } } development/idea-repository-core/src/main/java/com/ideabase/repository/core/index/factory/TermUsageAwareIndexFactory.java @@ -48,12 +48,16 @@ public class TermUsageAwareLuceneIndexReader implements LuceneIndexReader { LogManager.getLogger(TermUsageAwareLuceneIndexReader.class); private final LuceneIndexReader mBaseLuceneIndexReader; private final TermUsageService mTermUsageService; + private boolean mEnableAsyncTermStore = false; public TermUsageAwareLuceneIndexReader( final LuceneIndexReader pLuceneIndexReader, - final TermUsageService pTermUsageService) { + final TermUsageService pTermUsageService, + final boolean pEnableAsyncTermStore) { + mBaseLuceneIndexReader = pLuceneIndexReader; mTermUsageService = pTermUsageService; + mEnableAsyncTermStore = pEnableAsyncTermStore; } public void close() throws IOException { development/idea-repository-core/src/main/java/com/ideabase/repository/core/index/factory/TermUsageAwareLuceneIndexReader.java @@ -30,6 +30,8 @@ import org.apache.log4j.LogManager; import java.io.IOException; import java.io.PrintStream; import java.io.StringReader; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; import com.ideabase.repository.core.index.service.TermUsageService; import static com.ideabase.repository.core.helper.IndexHelper.isAcceptableFieldNameForTokenCollection; @@ -48,12 +50,16 @@ public class TermUsageAwareLuceneIndexWriter implements LuceneIndexWriter { private final LuceneIndexWriter mBaseLuceneIndexWriter; private final TermUsageService mTermUsageService; + private final Executor mAsyncTaskExecutor = Executors.newFixedThreadPool(5); + private boolean mEnableAsyncTermStore = false; public TermUsageAwareLuceneIndexWriter( final LuceneIndexWriter pBaseLuceneIndexWriter, - final TermUsageService pTermUsageService) { + final TermUsageService pTermUsageService, + final boolean pEnableAsyncTermStore) { mBaseLuceneIndexWriter = pBaseLuceneIndexWriter; mTermUsageService = pTermUsageService; + mEnableAsyncTermStore = pEnableAsyncTermStore; } public void addDocument(final Document pDocument) throws IOException { @@ -67,6 +73,14 @@ public class TermUsageAwareLuceneIndexWriter implements LuceneIndexWriter { parseTermsFromDocument(pDocument, pAnalyzer); } + public boolean isEnableAsyncTermStore() { + return mEnableAsyncTermStore; + } + + public void setEnableAsyncTermStore(final boolean pEnableAsyncTermStore) { + mEnableAsyncTermStore = pEnableAsyncTermStore; + } + /** * Parse terms from the specified document and store their hit count * through {@code TermUsageService} @@ -80,7 +94,8 @@ public class TermUsageAwareLuceneIndexWriter implements LuceneIndexWriter { final Field field = (Field) fieldObject; try { if (LOG.isDebugEnabled()) { - LOG.debug("PTFD: caculating tokens for - " + field.name() + " - " + isAcceptableFieldNameForTokenCollection(field.name())); + LOG.debug("caculating tokens for - " + field.name() + " - " + + isAcceptableFieldNameForTokenCollection(field.name())); } // retrieve associated item id final Field itemIdField = pDocument.getField(ObjectBase.INDEX_FIELD_ID); @@ -94,8 +109,9 @@ public class TermUsageAwareLuceneIndexWriter implements LuceneIndexWriter { if (tokenStream != null) { Token token = null; while ((token = tokenStream.next()) != null) { + // TODO: store token in async mode, need to use thread worker. final String tokenString = String.valueOf(token.termBuffer()).trim(); - mTermUsageService.storeTerm(tokenString, field.name(), itemId); + submitNewTermStoreRequest(tokenString, field.name(), itemId); } } } @@ -105,6 +121,20 @@ public class TermUsageAwareLuceneIndexWriter implements LuceneIndexWriter { } } + private void submitNewTermStoreRequest(final String pTokenString, + final String pFieldName, + final int pItemId) { + if (isEnableAsyncTermStore()) { + mAsyncTaskExecutor.execute(new Runnable() { + public void run() { + mTermUsageService.storeTerm(pTokenString, pFieldName, pItemId); + } + }); + } else { + mTermUsageService.storeTerm(pTokenString, pFieldName, pItemId); + } + } + public void addIndexes(final Directory[] pDirectories) throws IOException { mBaseLuceneIndexWriter.addIndexes(pDirectories); } development/idea-repository-core/src/main/java/com/ideabase/repository/core/index/factory/TermUsageAwareLuceneIndexWriter.java @@ -146,6 +146,7 @@ ]]></iterate> GROUP BY term ORDER BY count DESC + LIMIT #offset#, #max# </select> development/idea-repository-core/src/main/resources/config/database/mysql/Term.xml 2b2c55cc87fc000fec8258abbd1b81a2660d2c3f nhm tanveer hossain khan (hasan) hasan@somewherein.net http://github.com/we4tech/semantic-repository/commit/1aa2d4a74eda7332fa56e478fa6c7c7f8f282c73 1aa2d4a74eda7332fa56e478fa6c7c7f8f282c73 2009-03-18T13:49:07-07:00 2009-03-18T13:49:07-07:00 optimized search based tag cloud service search based tag cloud service is now more optimized, completely rewritten logic. added database native pagination for better performance. e07c517a287aa5c920f7f59a46a4e6ea248cf67f nhm tanveer hossain khan (hasan) hasan@somewherein.net