This repository has been archived by the owner on Nov 9, 2017. It is now read-only.
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Refactor Indexing Strategies for better performance.
Create an indexing strategy that performs less database queries and is able to index TextFlow targets faster.
- Loading branch information
Showing
6 changed files
with
202 additions
and
95 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
85 changes: 85 additions & 0 deletions
85
zanata-war/src/main/java/org/zanata/search/AbstractIndexingStrategy.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| package org.zanata.search; | ||
|
|
||
| import org.hibernate.Query; | ||
| import org.hibernate.ScrollableResults; | ||
| import org.hibernate.search.FullTextSession; | ||
|
|
||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| /** | ||
| * Base indexing strategy. | ||
| * | ||
| * @param <T> The type of object that this indexing strategy handles. | ||
| */ | ||
| @Slf4j | ||
| public abstract class AbstractIndexingStrategy<T> | ||
| { | ||
| private IndexerProcessHandle handle; | ||
| private int sessionClearBatchSize = 1000; | ||
| FullTextSession session; | ||
| Class<T> clazz; | ||
| ScrollableResults scrollableResults; | ||
|
|
||
|
|
||
| public AbstractIndexingStrategy(FullTextSession session, IndexerProcessHandle handle, Class<T> clazz) | ||
| { | ||
| this.session = session; | ||
| this.handle = handle; | ||
| this.clazz = clazz; | ||
| } | ||
|
|
||
| /** | ||
| * Performs the indexing. | ||
| */ | ||
| public void invoke() | ||
| { | ||
| int n = 0; | ||
| try | ||
| { | ||
| scrollableResults = getScrollableResults(session, clazz, n); | ||
| while (scrollableResults.next() && !handle.shouldStop()) | ||
| { | ||
| n++; | ||
| T entity = (T) scrollableResults.get(0); // index each element | ||
| session.index(entity); | ||
| handle.incrementProgress(1); | ||
| if (n % sessionClearBatchSize == 0) | ||
| { | ||
| log.info("periodic flush and clear for {} (n={})", clazz, n); | ||
| session.flushToIndexes(); // apply changes to indexes | ||
| session.clear(); // clear since the queue is processed | ||
| } | ||
| onEntityIndexed(n); | ||
| } | ||
| } | ||
| finally | ||
| { | ||
| if( scrollableResults != null ) | ||
| { | ||
| scrollableResults.close(); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Callback method that is called every time an entity is indexed. | ||
| * @param n The entity number that was indexed. | ||
| */ | ||
| protected abstract void onEntityIndexed(int n); | ||
|
|
||
| /** | ||
| * Returns the Scrollable results | ||
| * @param session Session used to query and index the entities | ||
| * @param clazz The type of entity to be returned by the Scrollable results | ||
| * @param firstResult | ||
| * @return | ||
| */ | ||
| protected abstract ScrollableResults getScrollableResults(FullTextSession session, Class<T> clazz, int firstResult); | ||
|
|
||
| /** | ||
| * Create a query which returns instances of clazz | ||
| * @param clazz The type of objects being returned by this query. | ||
| * @return | ||
| */ | ||
| protected abstract Query getQuery(FullTextSession session, Class<T> clazz); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
44 changes: 0 additions & 44 deletions
44
zanata-war/src/main/java/org/zanata/search/GenericClassIndexer.java
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
73 changes: 73 additions & 0 deletions
73
zanata-war/src/main/java/org/zanata/search/SimpleClassIndexingStrategy.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| /* | ||
| * Copyright 2010, Red Hat, Inc. and individual contributors as indicated by the | ||
| * @author tags. See the copyright.txt file in the distribution for a full | ||
| * listing of individual contributors. | ||
| * | ||
| * This is free software; you can redistribute it and/or modify it under the | ||
| * terms of the GNU Lesser General Public License as published by the Free | ||
| * Software Foundation; either version 2.1 of the License, or (at your option) | ||
| * any later version. | ||
| * | ||
| * This software is distributed in the hope that it will be useful, but WITHOUT | ||
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | ||
| * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more | ||
| * details. | ||
| * | ||
| * You should have received a copy of the GNU Lesser General Public License | ||
| * along with this software; if not, write to the Free Software Foundation, | ||
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA, or see the FSF | ||
| * site: http://www.fsf.org. | ||
| */ | ||
| package org.zanata.search; | ||
|
|
||
| import org.hibernate.Query; | ||
| import org.hibernate.ScrollMode; | ||
| import org.hibernate.ScrollableResults; | ||
| import org.hibernate.search.FullTextSession; | ||
|
|
||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| /** | ||
| * Indexing strategy that fetches all instances in a given class and indexes them. | ||
| * This class batches the fetching of the entities and might be a bit slower as it does | ||
| * not account for lazily loaded entity relationships. | ||
| * | ||
| * @author Carlos Munoz <a href="mailto:camunoz@redhat.com">camunoz@redhat.com</a> | ||
| */ | ||
| @Slf4j | ||
| public class SimpleClassIndexingStrategy<T> extends AbstractIndexingStrategy<T> | ||
| { | ||
|
|
||
| public static final int MAX_QUERY_ROWS = 5000; | ||
|
|
||
| public SimpleClassIndexingStrategy(FullTextSession session, IndexerProcessHandle handle, Class<T> clazz) | ||
| { | ||
| super(session, handle, clazz); | ||
| } | ||
|
|
||
| @Override | ||
| protected void onEntityIndexed(int n) | ||
| { | ||
| if (n % MAX_QUERY_ROWS == 0) | ||
| { | ||
| SimpleClassIndexingStrategy.log.info("restarting query for {} (n={})", clazz, n); | ||
| scrollableResults.close(); | ||
| scrollableResults = getScrollableResults(session, clazz, n); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| protected ScrollableResults getScrollableResults(FullTextSession session, Class<T> clazz, int firstResult) | ||
| { | ||
| Query query = getQuery(session, clazz); | ||
| query.setFirstResult(firstResult); | ||
| query.setMaxResults(MAX_QUERY_ROWS); | ||
| return query.scroll(ScrollMode.FORWARD_ONLY); | ||
| } | ||
|
|
||
| @Override | ||
| protected Query getQuery(FullTextSession session, Class<T> clazz) | ||
| { | ||
| return session.createQuery("from "+clazz.getName()); | ||
| } | ||
| } |