Skip to content
This repository has been archived by the owner on Nov 9, 2017. It is now read-only.

Commit

Permalink
Testing on performance with lucene copytrans
Browse files Browse the repository at this point in the history
  • Loading branch information
Alex Eng committed Apr 16, 2014
2 parents 7dd14b9 + 890c48d commit d9c1e84
Show file tree
Hide file tree
Showing 4 changed files with 32 additions and 34 deletions.
17 changes: 8 additions & 9 deletions zanata-war/src/main/java/org/zanata/service/impl/CopyTransServiceImpl.java
View file
Expand Up @@ -22,10 +22,6 @@

import java.util.List;

import lombok.AllArgsConstructor;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;

import org.jboss.seam.ScopeType;
import org.jboss.seam.annotations.In;
import org.jboss.seam.annotations.Name;
Expand All @@ -41,10 +37,13 @@
import org.zanata.model.HTextFlow;
import org.zanata.service.CopyTransService;
import org.zanata.service.LocaleService;

import com.google.common.base.Optional;
import com.google.common.base.Stopwatch;

import lombok.AllArgsConstructor;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;

@Name("copyTransServiceImpl")
@Scope(ScopeType.STATELESS)
@Slf4j
Expand All @@ -67,10 +66,10 @@ public class CopyTransServiceImpl implements CopyTransService {
* Copies previous matching translations for the given locale into a
* document. Translations are matching if their document id, textflow id and
* source content are identical, and their state is approved.
*
*
* The text flow revision for copied targets is set to the current text flow
* revision.
*
*
* @param document
* the document to copy translations into
* @param targetLocale
Expand All @@ -92,7 +91,7 @@ private void copyTransForDocumentLocale(HDocument document,
start += COPY_TRANS_BATCH_SIZE;
}
stopwatch.stop();
System.out.println("Total time:" + stopwatch);
System.out.println("CopyTrans complete:" + stopwatch);

// Advance the task handler if there is one
Optional<CopyTransTaskHandle> taskHandle =
Expand All @@ -108,7 +107,7 @@ private void copyTransForDocumentLocale(HDocument document,

/**
* Perform copy trans on a batch of text flows for a document.
*
*
* @param batchStart
* USE_HIBERNATE_SEARCH The text flow position to start copying.
* @param batchLength
Expand Down
7 changes: 2 additions & 5 deletions zanata-war/src/main/java/org/zanata/service/impl/CopyTransWorkFactory.java
View file
Expand Up @@ -20,24 +20,21 @@
*/
package org.zanata.service.impl;

import java.util.List;

import org.jboss.seam.ScopeType;
import org.jboss.seam.annotations.In;
import org.jboss.seam.annotations.Name;
import org.jboss.seam.annotations.Scope;
import org.zanata.dao.DocumentDAO;
import org.zanata.dao.ProjectDAO;
import org.zanata.dao.TextFlowTargetDAO;
import org.zanata.model.HCopyTransOptions;
import org.zanata.model.HDocument;
import org.zanata.model.HLocale;
import org.zanata.model.HTextFlow;
import org.zanata.service.LocaleService;
import org.zanata.service.TranslationFinder;
import org.zanata.service.ValidationService;
import org.zanata.service.VersionStateCache;

import java.util.List;

/**
* @author Sean Flanigan <a href="mailto:sflaniga@redhat.com">sflaniga@redhat.com</a>
*/
Expand Down
32 changes: 15 additions & 17 deletions zanata-war/src/main/java/org/zanata/service/impl/TranslationMemoryServiceImpl.java
View file
Expand Up @@ -22,8 +22,6 @@

package org.zanata.service.impl;

import static com.google.common.collect.Collections2.filter;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
Expand All @@ -32,8 +30,6 @@
import java.util.List;
import java.util.Map;

import lombok.extern.slf4j.Slf4j;

import org.apache.commons.lang.StringUtils;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.Term;
Expand Down Expand Up @@ -72,13 +68,15 @@
import org.zanata.webtrans.shared.model.TransMemoryQuery;
import org.zanata.webtrans.shared.model.TransMemoryResultItem;
import org.zanata.webtrans.shared.rpc.HasSearchType;

import com.google.common.base.Objects;
import com.google.common.base.Optional;
import com.google.common.base.Predicate;
import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;

import static com.google.common.collect.Collections2.filter;
import lombok.extern.slf4j.Slf4j;

/**
* @author Alex Eng <a href="mailto:aeng@redhat.com">aeng@redhat.com</a>
*/
Expand All @@ -92,6 +90,8 @@ public class TranslationMemoryServiceImpl implements TranslationMemoryService {

private static final Version LUCENE_VERSION = Version.LUCENE_29;

private boolean USE_LUCENE_SORT = true;

@Override
public TransMemoryDetails
getTransMemoryDetail(HLocale hLocale, HTextFlow tf) {
Expand All @@ -114,12 +114,10 @@ public class TranslationMemoryServiceImpl implements TranslationMemoryService {
msgContext, tft.getState(), username, tft.getLastChanged());
}

private boolean useLuceneSort = false;

/**
* This is used by CopyTrans, with ContentHash search in lucene. Returns
* first entry of the matches which sort by HTextFlowTarget.lastChanged DESC
*
*
* @param textFlow
* @param targetLocaleId
* @param sourceLocaleId
Expand All @@ -145,7 +143,7 @@ public Optional<HTextFlowTarget> searchBestMatchTransMemory(
return Optional.<HTextFlowTarget> absent();
}

if (!useLuceneSort) {
if (!USE_LUCENE_SORT) {
Collections.sort(matches, new Comparator<Object[]>() {
@Override
public int compare(Object[] o1, Object[] o2) {
Expand All @@ -163,7 +161,7 @@ public int compare(Object[] o1, Object[] o2) {
/**
* This is used by TMMerge. Returns first entry of the matches which sort by
* similarityPercent, sourceContents, and contents size.
*
*
* @param textFlow
* @param targetLocaleId
* @param sourceLocaleId
Expand Down Expand Up @@ -253,7 +251,7 @@ private TransMemoryQuery buildTMQuery(HTextFlow textFlow,
/**
* return match[0] = (float)score, match[1] = entity(HTextFlowTarget or
* TransMemoryUnit)
*
*
* @param targetLocaleId
* @param sourceLocaleId
* @param transMemoryQuery
Expand Down Expand Up @@ -536,7 +534,7 @@ private List<Object[]> getSearchResult(TransMemoryQuery query,
FullTextQuery ftQuery =
entityManager.createFullTextQuery(textQuery, entities);

if(useLuceneSort) {
if(USE_LUCENE_SORT) {
// sort desc by lastChanged of HTextFlowTarget
ftQuery.setSort(new Sort(
new SortField(IndexFieldLabels.LAST_CHANGED_FIELD,
Expand All @@ -557,7 +555,7 @@ private List<Object[]> getSearchResult(TransMemoryQuery query,
/**
* Generate the query to match all source contents in all the searchable
* indexes. (HTextFlowTarget and TransMemoryUnit)
*
*
* @param query
* @param sourceLocale
* @param targetLocale
Expand Down Expand Up @@ -591,7 +589,7 @@ private Query generateQuery(TransMemoryQuery query, LocaleId sourceLocale,
/**
* Generates the Hibernate Search Query that will search for
* {@link HTextFlowTarget} objects for matches.
*
*
* @param queryParams
* @param sourceLocale
* @param targetLocale
Expand Down Expand Up @@ -640,7 +638,7 @@ private Query generateTextFlowTargetQuery(TransMemoryQuery queryParams,

/**
* Build query for project, document and resId context
*
*
* @param queryParams
* @return
*/
Expand Down Expand Up @@ -723,7 +721,7 @@ private static TermQuery buildStateQuery(ContentState state) {
/**
* Generates the Hibernate Search Query that will search for
* {@link org.zanata.model.tm.TransMemoryUnit} objects for matches.
*
*
* @param sourceLocale
* @param targetLocale
* @param queryText
Expand Down Expand Up @@ -754,7 +752,7 @@ private Query generateTransMemoryQuery(LocaleId sourceLocale,
/**
* Joins a given set of queries into a single one with the specified
* occurrence condition.
*
*
* @param condition
* The occurrence condition all the joined queries will have.
* @param queries
Expand Down
10 changes: 7 additions & 3 deletions zanata-war/src/test/java/org/zanata/service/impl/CopyTransServiceUnitTest.java
View file
Expand Up @@ -20,6 +20,9 @@
*/
package org.zanata.service.impl;

import java.util.Arrays;
import java.util.List;

import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.testng.annotations.BeforeMethod;
Expand All @@ -39,9 +42,6 @@
import org.zanata.service.ValidationService;
import org.zanata.service.VersionStateCache;

import java.util.Arrays;
import java.util.List;

import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyListOf;
Expand Down Expand Up @@ -101,13 +101,16 @@ private void shouldUseProjectOptions(boolean useProjectOpts) throws Exception {
HProject proj = createProject(projSlug, projId, projOptions);
when(projectDAO.findById(projId, false)).thenReturn(proj);


String iterSlug = "iterslug";
boolean requireReview = true;
HProjectIteration iter = createIter(iterSlug, proj, requireReview);

List<HTextFlow> textFlows = Arrays.asList(new HTextFlow());
HDocument doc = createDoc(iter, textFlows);

when(documentDAO.findById(doc.getId())).thenReturn(doc);

HLocale de = new HLocale(LocaleId.DE);
List<HLocale> localeList = Arrays.asList(de);
when(localeServiceImpl.getSupportedLanguageByProjectIteration(projSlug, iterSlug)).thenReturn(localeList);
Expand Down Expand Up @@ -138,6 +141,7 @@ private void shouldUseProjectOptions(boolean useProjectOpts) throws Exception {
private HDocument createDoc(HProjectIteration iter,
List<HTextFlow> textFlows) {
HDocument doc = new HDocument();
doc.setId(9999L);
doc.setProjectIteration(iter);
doc.setTextFlows(textFlows);
return doc;
Expand Down

0 comments on commit d9c1e84

Please sign in to comment.