Skip to content
This repository has been archived by the owner on Nov 9, 2017. It is now read-only.

Commit

Permalink
Merge branch 'rhbz874342' into release
Browse files Browse the repository at this point in the history
  • Loading branch information
Alex Eng committed Nov 28, 2012
2 parents ddd1d18 + 07113ab commit 86e4afe
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 37 deletions.
67 changes: 49 additions & 18 deletions zanata-war/src/main/java/org/zanata/dao/TextFlowDAO.java
View file
Expand Up @@ -23,16 +23,17 @@
import java.util.ArrayList;
import java.util.List;


import org.apache.lucene.analysis.Analyzer;

import lombok.extern.slf4j.Slf4j;

import org.apache.commons.lang.StringUtils;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.MultiFieldQueryParser;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.Version;
import org.hibernate.Criteria;
import org.hibernate.Hibernate;
Expand All @@ -47,6 +48,7 @@
import org.jboss.seam.annotations.In;
import org.jboss.seam.annotations.Name;
import org.jboss.seam.annotations.Scope;
import org.zanata.common.ContentState;
import org.zanata.common.LocaleId;
import org.zanata.hibernate.search.IndexFieldLabels;
import org.zanata.hibernate.search.TextContainerAnalyzerDiscriminator;
Expand Down Expand Up @@ -229,30 +231,30 @@ protected static String buildSearchCondition(String searchString, String alias)
/**
* Using id list and source index
*/
public List<Object[]> getSearchResult(TransMemoryQuery query, List<Long> translatedIds, LocaleId locale, final int maxResult) throws ParseException
public List<Object[]> getSearchResult(TransMemoryQuery query, List<Long> translatedIds, LocaleId sourceLocale, LocaleId targetLocale, final int maxResult) throws ParseException
{
return getSearchResult(query, translatedIds, locale, maxResult, false);
return getSearchResult(query, translatedIds, sourceLocale, targetLocale, maxResult, false);
}

/**
* Using target index (to be phased out)
*/
public List<Object[]> getSearchResult(TransMemoryQuery query, LocaleId locale, final int maxResult) throws ParseException
public List<Object[]> getSearchResult(TransMemoryQuery query, LocaleId sourceLocale, LocaleId targetLocale, final int maxResult) throws ParseException
{
return getSearchResult(query, null, locale, maxResult, true);
return getSearchResult(query, null, sourceLocale, targetLocale, maxResult, true);
}

/**
*
* @param query
* @param translatedIds ignored if useTargetIndex is true
* @param locale
* @param sourceLocale
* @param maxResult
* @param useTargetIndex
* @return
* @throws ParseException
*/
private List<Object[]> getSearchResult(TransMemoryQuery query, List<Long> translatedIds, LocaleId locale, final int maxResult, boolean useTargetIndex) throws ParseException
private List<Object[]> getSearchResult(TransMemoryQuery query, List<Long> translatedIds, LocaleId sourceLocale, LocaleId targetLocale, final int maxResult, boolean useTargetIndex) throws ParseException
{
String queryText = null;
String[] multiQueryText = null;
Expand Down Expand Up @@ -305,12 +307,12 @@ private List<Object[]> getSearchResult(TransMemoryQuery query, List<Long> transl
FullTextQuery ftQuery;
if (useTargetIndex)
{
org.apache.lucene.search.Query textQuery = generateTextQuery(query, locale, queryText, multiQueryText, IndexFieldLabels.TF_CONTENT_FIELDS);
org.apache.lucene.search.Query textQuery = generateQuery(query, sourceLocale, targetLocale, queryText, multiQueryText, IndexFieldLabels.TF_CONTENT_FIELDS, useTargetIndex);
ftQuery = entityManager.createFullTextQuery(textQuery, HTextFlowTarget.class);
}
else
{
org.apache.lucene.search.Query textQuery = generateTextQuery(query, locale, queryText, multiQueryText, IndexFieldLabels.CONTENT_FIELDS);
org.apache.lucene.search.Query textQuery = generateQuery(query, sourceLocale, targetLocale, queryText, multiQueryText, IndexFieldLabels.CONTENT_FIELDS, useTargetIndex);
ftQuery = entityManager.createFullTextQuery(textQuery, HTextFlow.class);
ftQuery.enableFullTextFilter("textFlowFilter").setParameter("ids", translatedIds);
}
Expand All @@ -322,11 +324,24 @@ private List<Object[]> getSearchResult(TransMemoryQuery query, List<Long> transl
return matches;
}

private org.apache.lucene.search.Query generateTextQuery(TransMemoryQuery query, LocaleId locale, String queryText, String[] multiQueryText, String[] contentFields) throws ParseException
/**
* Generate HTextFlowTarget query with matching HTextFlow contents,
* HTextFlowTarget locale, HTextFlowTarget state = Approved
*
* @param query
* @param sourceLocale
* @param targetLocale
* @param queryText
* @param multiQueryText
* @param contentFields
* @return
* @throws ParseException
*/
private org.apache.lucene.search.Query generateQuery(TransMemoryQuery query, LocaleId sourceLocale, LocaleId targetLocale, String queryText, String[] multiQueryText, String contentFields[], boolean useTargetIndex) throws ParseException
{
org.apache.lucene.search.Query textQuery;
org.apache.lucene.search.Query contentQuery;
// Analyzer determined by the language
String analyzerDefName = TextContainerAnalyzerDiscriminator.getAnalyzerDefinitionName( locale.getId() );
String analyzerDefName = TextContainerAnalyzerDiscriminator.getAnalyzerDefinitionName(sourceLocale.getId());
Analyzer analyzer = entityManager.getSearchFactory().getAnalyzer(analyzerDefName);

if (query.getSearchType() == SearchType.FUZZY_PLURAL)
Expand All @@ -339,14 +354,30 @@ private org.apache.lucene.search.Query generateTextQuery(TransMemoryQuery query,
String[] searchFields = new String[queriesSize];
System.arraycopy(contentFields, 0, searchFields, 0, queriesSize);

textQuery = MultiFieldQueryParser.parse(LUCENE_VERSION, multiQueryText, searchFields, analyzer);
contentQuery = MultiFieldQueryParser.parse(LUCENE_VERSION, multiQueryText, searchFields, analyzer);
}
else
{
MultiFieldQueryParser parser = new MultiFieldQueryParser(LUCENE_VERSION, contentFields, analyzer);
textQuery = parser.parse(queryText);
contentQuery = parser.parse(queryText);
}

if (useTargetIndex)
{
TermQuery localeQuery = new TermQuery(new Term(IndexFieldLabels.LOCALE_ID_FIELD, targetLocale.getId()));
TermQuery stateQuery = new TermQuery(new Term(IndexFieldLabels.CONTENT_STATE_FIELD, ContentState.Approved.toString()));

BooleanQuery targetQuery = new BooleanQuery();
targetQuery.add(contentQuery, Occur.MUST);
targetQuery.add(localeQuery, Occur.MUST);
targetQuery.add(stateQuery, Occur.MUST);

return targetQuery;
}
else
{
return contentQuery;
}
return textQuery;
}

public int getTotalWords()
Expand Down
23 changes: 10 additions & 13 deletions zanata-war/src/main/java/org/zanata/webtrans/server/rpc/GetTransMemoryHandler.java
View file
Expand Up @@ -36,7 +36,6 @@
import org.jboss.seam.annotations.In;
import org.jboss.seam.annotations.Name;
import org.jboss.seam.annotations.Scope;
import org.zanata.common.ContentState;
import org.zanata.common.EntityStatus;
import org.zanata.common.LocaleId;
import org.zanata.dao.TextFlowDAO;
Expand Down Expand Up @@ -102,21 +101,21 @@ protected ArrayList<TransMemoryResultItem> searchTransMemory(HLocale targetLocal
List<Object[]> matches;
if (useTargetIndex)
{
matches = textFlowDAO.getSearchResult(transMemoryQuery, sourceLocaleId, MAX_RESULTS);
matches = textFlowDAO.getSearchResult(transMemoryQuery, sourceLocaleId, targetLocale.getLocaleId(), MAX_RESULTS);
}
else
{
// FIXME this won't scale well(findIdsWithTransliations will scan the entire table each time)
List<Long> idsWithTranslations = textFlowDAO.findIdsWithTranslations(targetLocale.getLocaleId());
matches = textFlowDAO.getSearchResult(transMemoryQuery, idsWithTranslations, sourceLocaleId, MAX_RESULTS);
matches = textFlowDAO.getSearchResult(transMemoryQuery, idsWithTranslations, sourceLocaleId, targetLocale.getLocaleId(), MAX_RESULTS);
}

Map<TMKey, TransMemoryResultItem> matchesMap = new LinkedHashMap<TMKey, TransMemoryResultItem>(matches.size());
for (Object[] match : matches)
{
if (useTargetIndex)
{
processTargetIndexMatch(targetLocale, transMemoryQuery, matchesMap, match);
processTargetIndexMatch(transMemoryQuery, matchesMap, match);
}
else
{
Expand All @@ -143,15 +142,15 @@ protected ArrayList<TransMemoryResultItem> searchTransMemory(HLocale targetLocal
return results;
}

private void processTargetIndexMatch(HLocale targetLocale, TransMemoryQuery transMemoryQuery, Map<TMKey, TransMemoryResultItem> matchesMap, Object[] match)
private void processTargetIndexMatch(TransMemoryQuery transMemoryQuery, Map<TMKey, TransMemoryResultItem> matchesMap, Object[] match)
{
HTextFlowTarget textFlowTarget = (HTextFlowTarget) match[1];
if (isInvalidResult(textFlowTarget, targetLocale))
if (!isValidResult(textFlowTarget))
{
return;
}
ArrayList<String> textFlowContents = new ArrayList<String>(textFlowTarget.getTextFlow().getContents());
ArrayList<String> targetContents = new ArrayList<String>(textFlowTarget.getTextFlow().getTargets().get(targetLocale.getId()).getContents());
ArrayList<String> targetContents = new ArrayList<String>(textFlowTarget.getContents());
addOrIncrementResultItem(transMemoryQuery, matchesMap, match, textFlowTarget, textFlowContents, targetContents);
}

Expand Down Expand Up @@ -186,23 +185,21 @@ private void addOrIncrementResultItem(TransMemoryQuery transMemoryQuery, Map<TMK
item.addSourceId(textFlowTarget.getTextFlow().getId());
}

private static boolean isInvalidResult(HTextFlowTarget textFlowTarget, HLocale hLocale)
private static boolean isValidResult(HTextFlowTarget textFlowTarget)
{
if (textFlowTarget == null)
{
return true;
return false;
}
else
{
HProjectIteration projectIteration = textFlowTarget.getTextFlow().getDocument().getProjectIteration();
if (projectIteration.getStatus() == EntityStatus.OBSOLETE || projectIteration.getProject().getStatus() == EntityStatus.OBSOLETE)
{
return true;
return false;
}
}
HTextFlowTarget target = textFlowTarget.getTextFlow().getTargets().get(hLocale.getId());
// double check in case of caching issues
return target == null || target.getState() != ContentState.Approved;
return true;
}


Expand Down
18 changes: 12 additions & 6 deletions zanata-war/src/test/java/org/zanata/webtrans/server/rpc/GetTransMemoryHandlerTest.java
View file
@@ -1,5 +1,14 @@
package org.zanata.webtrans.server.rpc;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.mockito.Matchers.anyList;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import java.util.List;

import org.apache.lucene.queryParser.ParseException;
Expand All @@ -26,9 +35,6 @@

import com.google.common.collect.Lists;

import static org.hamcrest.MatcherAssert.*;
import static org.mockito.Mockito.*;

/**
* @author Patrick Huang <a href="mailto:pahuang@redhat.com">pahuang@redhat.com</a>
*/
Expand Down Expand Up @@ -79,7 +85,7 @@ public void testExecute() throws Exception
HTextFlowTarget tmMatch1 = getEm().find(HTextFlowTarget.class, 60L);
HTextFlowTarget tmMatch2 = getEm().find(HTextFlowTarget.class, 62L);
List<Object[]> matches = Lists.newArrayList(new Object[] {1.0F, tmMatch1}, new Object[] {1.1F, tmMatch2});
doReturn(matches).when(textFlowDAOSpy).getSearchResult(eq(query), eq(sourceLocaleId), eq(25));
doReturn(matches).when(textFlowDAOSpy).getSearchResult(eq(query), eq(sourceLocaleId), eq(targetLocaleId), eq(25));
GetTranslationMemory action = new GetTranslationMemory(query, targetLocaleId, sourceLocaleId);

// When:
Expand All @@ -101,7 +107,7 @@ public void searchReturnNotApprovedResult() throws Exception
TransMemoryQuery query = new TransMemoryQuery(Lists.newArrayList("file removed"), HasSearchType.SearchType.FUZZY_PLURAL);
HTextFlow tmMatch1 = getEm().find(HTextFlowTarget.class, 61L).getTextFlow();
List<Object[]> matches = Lists.newArrayList(new Object[] {1.0F, tmMatch1}, new Object[] {1.1F, null});
doReturn(matches).when(textFlowDAOSpy).getSearchResult(eq(query), anyList(), eq(sourceLocaleId), eq(10));
doReturn(matches).when(textFlowDAOSpy).getSearchResult(eq(query), anyList(), eq(sourceLocaleId), eq(targetLocaleId), eq(10));
GetTranslationMemory action = new GetTranslationMemory(query, targetLocaleId, sourceLocaleId);

// When:
Expand All @@ -118,7 +124,7 @@ public void whenThereAreParseException() throws Exception
{
// Given: hibernate search can not parse query
TransMemoryQuery query = new TransMemoryQuery(Lists.newArrayList("file removed"), HasSearchType.SearchType.FUZZY_PLURAL);
doThrow(new ParseException("bad token")).when(textFlowDAOSpy).getSearchResult(eq(query), eq(sourceLocaleId), eq(25));
doThrow(new ParseException("bad token")).when(textFlowDAOSpy).getSearchResult(eq(query), eq(sourceLocaleId), eq(targetLocaleId), eq(25));
GetTranslationMemory action = new GetTranslationMemory(query, targetLocaleId, sourceLocaleId);

// When:
Expand Down

0 comments on commit 86e4afe

Please sign in to comment.