diff --git a/zanata-war/src/main/java/org/zanata/service/TranslationMemoryService.java b/zanata-war/src/main/java/org/zanata/service/TranslationMemoryService.java index d4c887143d..9ad090615a 100644 --- a/zanata-war/src/main/java/org/zanata/service/TranslationMemoryService.java +++ b/zanata-war/src/main/java/org/zanata/service/TranslationMemoryService.java @@ -47,6 +47,11 @@ Optional searchBestMatchTransMemory(HTextFlow textFlow, LocaleId targetLocaleId, LocaleId sourceLocaleId, boolean checkContext, boolean checkDocument, boolean checkProject); + Optional searchBestMatchTransMemory( + HTextFlow textFlow, LocaleId targetLocaleId, + LocaleId sourceLocaleId, boolean checkContext, + boolean checkDocument, boolean checkProject, int thresholdPercent); + List searchTransMemory(LocaleId targetLocaleId, LocaleId sourceLocaleId, TransMemoryQuery transMemoryQuery); } diff --git a/zanata-war/src/main/java/org/zanata/service/impl/TransMemoryMergeServiceImpl.java b/zanata-war/src/main/java/org/zanata/service/impl/TransMemoryMergeServiceImpl.java index f9b5d4ba6d..fc7f8ab2e7 100644 --- a/zanata-war/src/main/java/org/zanata/service/impl/TransMemoryMergeServiceImpl.java +++ b/zanata-war/src/main/java/org/zanata/service/impl/TransMemoryMergeServiceImpl.java @@ -20,10 +20,8 @@ */ package org.zanata.service.impl; -import static com.google.common.collect.Collections2.filter; import static org.zanata.service.SecurityService.TranslationAction.MODIFY; -import java.util.Collection; import java.util.List; import java.util.Map; @@ -50,15 +48,13 @@ import org.zanata.service.TranslationService; import org.zanata.webtrans.server.rpc.TransMemoryMergeStatusResolver; import org.zanata.webtrans.shared.model.TransMemoryDetails; -import org.zanata.webtrans.shared.model.TransMemoryQuery; import org.zanata.webtrans.shared.model.TransMemoryResultItem; import org.zanata.webtrans.shared.model.TransUnitUpdateRequest; -import org.zanata.webtrans.shared.rpc.HasSearchType; import org.zanata.webtrans.shared.rpc.MergeRule; import org.zanata.webtrans.shared.rpc.TransMemoryMerge; import org.zanata.webtrans.shared.rpc.TransUnitUpdated; -import com.google.common.base.Predicate; +import com.google.common.base.Optional; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -89,13 +85,10 @@ public class TransMemoryMergeServiceImpl implements TransMemoryMergeService { @In private TranslationService translationServiceImpl; - private static final TransMemoryResultItem NULL_OBJECT = - new TransMemoryResultItem(null, null, null, 0, 0); - @Override public List executeMerge( TransMemoryMerge action) throws ActionException { - HLocale hLocale = + HLocale targetLocale = localeServiceImpl.getByLocaleId(action.getWorkspaceId() .getLocaleId()); @@ -109,14 +102,10 @@ public List executeMerge( textFlowDAO .findByIdList(Lists.newArrayList(requestMap.keySet())); - TransMemoryAboveThresholdPredicate predicate = - new TransMemoryAboveThresholdPredicate( - action.getThresholdPercent()); - List updateRequests = Lists.newArrayList(); for (HTextFlow hTextFlow : hTextFlows) { HTextFlowTarget hTextFlowTarget = - hTextFlow.getTargets().get(hLocale.getId()); + hTextFlow.getTargets().get(targetLocale.getId()); // TODO rhbz953734 - TM getUpdateRequests won't override Translated // to Approved @@ -128,35 +117,26 @@ public List executeMerge( continue; } - TransMemoryQuery.Condition project = - new TransMemoryQuery.Condition( - action.getDifferentProjectRule() == MergeRule.REJECT, - hTextFlow.getDocument().getProjectIteration() - .getProject().getSlug()); - TransMemoryQuery.Condition document = - new TransMemoryQuery.Condition( - action.getDifferentDocumentRule() == MergeRule.REJECT, - hTextFlow.getDocument().getDocId()); - TransMemoryQuery.Condition res = - new TransMemoryQuery.Condition( - action.getDifferentContextRule() == MergeRule.REJECT, - hTextFlow.getResId()); - - List tmResults = - translationMemoryServiceImpl.searchTransMemory(hLocale - .getLocaleId(), hTextFlow.getDocument().getLocale() - .getLocaleId(), - new TransMemoryQuery(hTextFlow.getContents(), - HasSearchType.SearchType.FUZZY_PLURAL, - project, document, res)); - - TransMemoryResultItem tmResult = - findTMAboveThreshold(tmResults, predicate, hTextFlow, + boolean checkContext = + action.getDifferentContextRule() == MergeRule.REJECT; + + boolean checkDocument = + action.getDifferentDocumentRule() == MergeRule.REJECT; + + boolean checkProject = + action.getDifferentProjectRule() == MergeRule.REJECT; + + Optional tmResult = + translationMemoryServiceImpl.searchBestMatchTransMemory( + hTextFlow, targetLocale.getLocaleId(), hTextFlow + .getDocument().getLocale().getLocaleId(), + checkContext, checkDocument, checkProject, action.getThresholdPercent()); - TransUnitUpdateRequest request = - createRequest(action, hLocale, requestMap, hTextFlow, - tmResult, hTextFlowTarget); - if (request != null) { + + if (tmResult.isPresent()) { + TransUnitUpdateRequest request = + createRequest(action, targetLocale, requestMap, + hTextFlow, tmResult.get(), hTextFlowTarget); updateRequests.add(request); } } @@ -195,9 +175,6 @@ private TransUnitUpdateRequest createRequest(TransMemoryMerge action, HLocale hLocale, Map requestMap, HTextFlow hTextFlowToBeFilled, TransMemoryResultItem tmResult, HTextFlowTarget oldTarget) { - if (tmResult == NULL_OBJECT) { - return null; - } Long tmSourceId = tmResult.getSourceIdList().get(0); ContentState statusToSet; @@ -236,69 +213,16 @@ private TransUnitUpdateRequest createRequest(TransMemoryMerge action, } private static String buildTargetComment(TransMemoryDetails tmDetail) { - return new StringBuilder( - "auto translated by TM merge from ") + return new StringBuilder("auto translated by TM merge from ") .append("project: ").append(tmDetail.getProjectName()) .append(", version: ").append(tmDetail.getIterationName()) .append(", DocId: ").append(tmDetail.getDocId()).toString(); } private static String buildTargetComment(TransMemoryUnit tu) { - return new StringBuilder( - "auto translated by TM merge from ") + return new StringBuilder("auto translated by TM merge from ") .append("translation memory: ") .append(tu.getTranslationMemory().getSlug()) .append(", unique id: ").append(tu.getUniqueId()).toString(); } - - private static class TransMemoryAboveThresholdPredicate implements - Predicate { - private final int approvedThreshold; - - public TransMemoryAboveThresholdPredicate(int approvedThreshold) { - this.approvedThreshold = approvedThreshold; - } - - @Override - public boolean apply(TransMemoryResultItem tmResult) { - return (int) tmResult.getSimilarityPercent() >= approvedThreshold; - } - } - - private static TransMemoryResultItem findTMAboveThreshold( - List tmResults, - TransMemoryAboveThresholdPredicate predicate, - final HTextFlow hTextFlow, int thresholdPercent) { - - Collection aboveThreshold; - - if (thresholdPercent == 100) { - aboveThreshold = - filter(tmResults, - new ContentsIdenticalPredicate(hTextFlow - .getContents())); - } else { - aboveThreshold = filter(tmResults, predicate); - } - if (aboveThreshold.size() > 0) { - return aboveThreshold.iterator().next(); - } else { - return NULL_OBJECT; - } - } - - private static class ContentsIdenticalPredicate implements - Predicate { - private final List sourceContents; - - public ContentsIdenticalPredicate(List sourceContents) { - this.sourceContents = sourceContents; - } - - @Override - public boolean apply(TransMemoryResultItem input) { - return input.getSourceContents().equals(sourceContents); - } - } - } diff --git a/zanata-war/src/main/java/org/zanata/service/impl/TranslationMemoryServiceImpl.java b/zanata-war/src/main/java/org/zanata/service/impl/TranslationMemoryServiceImpl.java index 205a68292a..3a6d6dd697 100644 --- a/zanata-war/src/main/java/org/zanata/service/impl/TranslationMemoryServiceImpl.java +++ b/zanata-war/src/main/java/org/zanata/service/impl/TranslationMemoryServiceImpl.java @@ -22,13 +22,18 @@ package org.zanata.service.impl; +import static com.google.common.collect.Collections2.filter; + import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import lombok.extern.slf4j.Slf4j; + import org.apache.commons.lang.StringUtils; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.Term; @@ -65,12 +70,14 @@ import org.zanata.webtrans.shared.model.TransMemoryQuery; import org.zanata.webtrans.shared.model.TransMemoryResultItem; import org.zanata.webtrans.shared.rpc.HasSearchType; + +import com.google.common.base.Function; import com.google.common.base.Objects; import com.google.common.base.Optional; +import com.google.common.base.Predicate; +import com.google.common.collect.Collections2; import com.google.common.collect.Lists; -import lombok.extern.slf4j.Slf4j; - /** * @author Alex Eng aeng@redhat.com */ @@ -108,48 +115,81 @@ public class TranslationMemoryServiceImpl implements TranslationMemoryService { tft.getState(), username, tft.getLastChanged()); } + /** + * This is used by CopyTrans, with ContentHash search in lucene. Returns + * first entry of the matches which sort by HTextFlowTarget.lastChanged DESC + * + * @param textFlow + * @param targetLocaleId + * @param sourceLocaleId + * @param checkContext + * @param checkDocument + * @param checkProject + */ @Override public Optional searchBestMatchTransMemory( HTextFlow textFlow, LocaleId targetLocaleId, LocaleId sourceLocaleId, boolean checkContext, boolean checkDocument, boolean checkProject) { - TransMemoryQuery.Condition project = - new TransMemoryQuery.Condition(checkProject, textFlow - .getDocument().getProjectIteration().getProject() - .getSlug()); - TransMemoryQuery.Condition document = - new TransMemoryQuery.Condition(checkDocument, textFlow - .getDocument().getDocId()); - TransMemoryQuery.Condition res = - new TransMemoryQuery.Condition(checkContext, - textFlow.getResId()); - TransMemoryQuery query = - new TransMemoryQuery(textFlow.getContentHash(), - HasSearchType.SearchType.CONTENT_HASH, project, - document, res); + buildTMQuery(textFlow, HasSearchType.SearchType.CONTENT_HASH, + checkContext, checkDocument, checkProject); List matches = findMatchingTranslation(targetLocaleId, sourceLocaleId, query, null, HTextFlowTarget.class); - List matchedTarget = Lists.newArrayList(); + List matchedTargets = + Lists.newArrayList(Collections2.transform(matches, + new Function() { + @Override + public HTextFlowTarget apply(Object[] input) { + return (HTextFlowTarget) input[1]; + } + })); - for (Object[] match : matches) { - Object entity = match[1]; - HTextFlowTarget textFlowTarget = (HTextFlowTarget) entity; - if (isValidResult(textFlowTarget)) { - matchedTarget.add(textFlowTarget); - } - } - - if (matchedTarget.isEmpty()) { + if (matchedTargets.isEmpty()) { return Optional. absent(); } - Collections.sort(matchedTarget, HTextFlowTargetComparator.COMPARATOR); - return Optional.of((HTextFlowTarget) matchedTarget.get(0)); + Collections.sort(matchedTargets, HTextFlowTargetComparator.COMPARATOR); + return Optional.of((HTextFlowTarget) matchedTargets.get(0)); + } + + /** + * This is used by TMMerge. Returns first entry of the matches which sort by + * similarityPercent, sourceContents, and contents size. + * + * @param textFlow + * @param targetLocaleId + * @param sourceLocaleId + * @param checkContext + * @param checkDocument + * @param checkProject + * @param thresholdPercent + */ + @Override + public Optional searchBestMatchTransMemory( + HTextFlow textFlow, LocaleId targetLocaleId, + LocaleId sourceLocaleId, boolean checkContext, + boolean checkDocument, boolean checkProject, int thresholdPercent) { + + TransMemoryQuery query = + buildTMQuery(textFlow, HasSearchType.SearchType.FUZZY_PLURAL, + checkContext, checkDocument, checkProject); + + List tmResults = + searchTransMemory(targetLocaleId, sourceLocaleId, query); + + TransMemoryResultItem match = + findTMAboveThreshold(tmResults, textFlow.getContents(), + thresholdPercent); + + if (match == null) { + return Optional. absent(); + } + return Optional.of((TransMemoryResultItem) match); } @Override @@ -173,10 +213,51 @@ public List searchTransMemory( return results; } + private Optional findTMAboveThreshold( + List tmResults, + final List sourceContents, int thresholdPercent) { + + Collection aboveThreshold; + + if (thresholdPercent == 100) { + aboveThreshold = + filter(tmResults, new ContentsIdenticalPredicate( + sourceContents)); + } else { + aboveThreshold = + filter(tmResults, new TransMemoryAboveThresholdPredicate( + thresholdPercent)); + } + + if (aboveThreshold.isEmpty()) { + return Optional. absent(); + } + return Optional.of((TransMemoryResultItem) aboveThreshold.iterator() + .next()); + } + + private TransMemoryQuery buildTMQuery(HTextFlow textFlow, + HasSearchType.SearchType searchType, boolean checkContext, + boolean checkDocument, boolean checkProject) { + TransMemoryQuery.Condition project = + new TransMemoryQuery.Condition(checkProject, textFlow + .getDocument().getProjectIteration().getProject() + .getSlug()); + TransMemoryQuery.Condition document = + new TransMemoryQuery.Condition(checkDocument, textFlow + .getDocument().getDocId()); + TransMemoryQuery.Condition res = + new TransMemoryQuery.Condition(checkContext, + textFlow.getResId()); + + return new TransMemoryQuery(textFlow.getContentHash(), searchType, + project, document, res); + } + /** * return match[0] = (float)score, match[1] = entity(HTextFlowTarget or * TransMemoryUnit) - * + * * @param targetLocaleId * @param sourceLocaleId * @param transMemoryQuery @@ -191,6 +272,23 @@ private List findMatchingTranslation(LocaleId targetLocaleId, results = getSearchResult(transMemoryQuery, sourceLocaleId, targetLocaleId, maxResults, entities); + + // filter out invalid targets(obsolete project or version) + Collection filtered = + Collections2.filter(results, new Predicate() { + @Override + public boolean apply(Object[] input) { + Object entity = input[1]; + if (entity instanceof HTextFlowTarget) { + HTextFlowTarget target = + (HTextFlowTarget) entity; + return !isValidResult(target) ? false : true; + } + return true; + } + }); + results = Lists.newArrayList(filtered); + } catch (ParseException e) { if (transMemoryQuery.getSearchType() == HasSearchType.SearchType.RAW) { // TODO tell the user @@ -209,9 +307,7 @@ private void processIndexMatch(TransMemoryQuery transMemoryQuery, Object entity = match[1]; if (entity instanceof HTextFlowTarget) { HTextFlowTarget textFlowTarget = (HTextFlowTarget) entity; - if (!isValidResult(textFlowTarget)) { - return; - } + ArrayList textFlowContents = Lists.newArrayList(textFlowTarget.getTextFlow() .getContents()); @@ -304,7 +400,7 @@ private void addOrIncrementResultItem(TransMemoryQuery transMemoryQuery, item.addSourceId(sourceId); } - private static boolean isValidResult(HTextFlowTarget textFlowTarget) { + private boolean isValidResult(HTextFlowTarget textFlowTarget) { if (textFlowTarget == null || !textFlowTarget.getState().isTranslated()) { return false; } else { @@ -479,7 +575,7 @@ private List getSearchResult(TransMemoryQuery query, /** * Generate the query to match all source contents in all the searchable * indexes. (HTextFlowTarget and TransMemoryUnit) - * + * * @param query * @param sourceLocale * @param targetLocale @@ -513,7 +609,7 @@ private Query generateQuery(TransMemoryQuery query, LocaleId sourceLocale, /** * Generates the Hibernate Search Query that will search for * {@link HTextFlowTarget} objects for matches. - * + * * @param queryParams * @param sourceLocale * @param targetLocale @@ -553,7 +649,7 @@ private Query generateTextFlowTargetQuery(TransMemoryQuery queryParams, /** * Build query for project, document and resId context - * + * * @param queryParams * @return */ @@ -636,7 +732,7 @@ private static TermQuery buildStateQuery(ContentState state) { /** * Generates the Hibernate Search Query that will search for * {@link org.zanata.model.tm.TransMemoryUnit} objects for matches. - * + * * @param sourceLocale * @param targetLocale * @param queryText @@ -667,7 +763,7 @@ private Query generateTransMemoryQuery(LocaleId sourceLocale, /** * Joins a given set of queries into a single one with the specified * occurrence condition. - * + * * @param condition * The occurrence condition all the joined queries will have. * @param queries @@ -682,4 +778,33 @@ private static Query join(BooleanClause.Occur condition, Query... queries) { } return joinedQuery; } + + private static class TransMemoryAboveThresholdPredicate implements + Predicate { + private final int approvedThreshold; + + public TransMemoryAboveThresholdPredicate(int approvedThreshold) { + this.approvedThreshold = approvedThreshold; + } + + @Override + public boolean apply(TransMemoryResultItem tmResult) { + return (int) tmResult.getSimilarityPercent() >= approvedThreshold; + } + } + + private static class ContentsIdenticalPredicate implements + Predicate { + private final List sourceContents; + + public ContentsIdenticalPredicate(List sourceContents) { + this.sourceContents = sourceContents; + } + + @Override + public boolean apply(TransMemoryResultItem input) { + return input.getSourceContents().equals(sourceContents); + } + } + } diff --git a/zanata-war/src/test/java/org/zanata/service/impl/TransMemoryMergeServiceImplTest.java b/zanata-war/src/test/java/org/zanata/service/impl/TransMemoryMergeServiceImplTest.java index 78c7b48047..f39dc0b7cd 100644 --- a/zanata-war/src/test/java/org/zanata/service/impl/TransMemoryMergeServiceImplTest.java +++ b/zanata-war/src/test/java/org/zanata/service/impl/TransMemoryMergeServiceImplTest.java @@ -247,11 +247,16 @@ public void canAutoTranslateIfHasTMAboveThreshold() throws ActionException { tmResultSource)).thenReturn(tmDetail()); when( - translationMemoryService.searchTransMemory( +// translationMemoryService.searchTransMemory( +// targetLocale.getLocaleId(), sourceLocale.getLocaleId(), +// tmQuery)).thenReturn( +// newArrayList(mostSimilarTM, tmResult(12L, 90), +// tmResult(13L, 80))); + + translationMemoryService.searchBestMatchTransMemory(hTextFlow, targetLocale.getLocaleId(), sourceLocale.getLocaleId(), - tmQuery)).thenReturn( - newArrayList(mostSimilarTM, tmResult(12L, 90), - tmResult(13L, 80))); + false, false, false, action.getThresholdPercent())) + .thenReturn(null); // When: execute the action transMemoryMergeService.executeMerge(action);