Skip to content
This repository has been archived by the owner on Nov 9, 2017. It is now read-only.

Commit

Permalink
Merge branch 'release' into integration/master
Browse files Browse the repository at this point in the history
Conflicts:
	zanata-client-commands/src/main/java/org/zanata/client/commands/push/PushCommand.java
  • Loading branch information
Patrick Huang committed Dec 23, 2014
2 parents 93fb090 + 29cbd7e commit 1fbfeac
Show file tree
Hide file tree
Showing 5 changed files with 180 additions and 55 deletions.
Expand Up @@ -31,6 +31,7 @@
import java.util.HashSet;
import java.util.Set;

import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import org.xml.sax.InputSource;
import org.zanata.adapter.po.PoReader2;
Expand All @@ -45,6 +46,7 @@

public abstract class AbstractGettextPushStrategy extends AbstractPushStrategy {
private PoReader2 poReader = new PoReader2();
protected Set<String> localSrcDocNames;

public AbstractGettextPushStrategy() {
super(new StringSet("comment;gettext"), ".pot");
Expand All @@ -63,7 +65,10 @@ public Set<String> findDocNames(File srcDir, ImmutableList<String> includes,
ImmutableList<String> excludes, boolean useDefaultExclude,
boolean caseSensitive, boolean excludeLocaleFilenames)
throws IOException {
Set<String> localDocNames = new HashSet<String>();
if (localSrcDocNames != null) {
return localSrcDocNames;
}
localSrcDocNames = new HashSet<>();

// populate localDocNames by looking in pot directory, ignore
// excludeLocale option
Expand All @@ -73,14 +78,35 @@ public Set<String> findDocNames(File srcDir, ImmutableList<String> includes,

for (String potName : srcFiles) {
String docName = removeExtension(potName);
localDocNames.add(docName);
localSrcDocNames.add(docName);
}
checkSrcFileNames(getOpts().getProjectType(), srcFiles,
getOpts().isInteractiveMode());
return localDocNames;
return localSrcDocNames;
}

protected Set<String> getSrcDocNames() {
try {
return findDocNames(getOpts().getSrcDir(),
getOpts().getIncludes(),
getOpts().getExcludes(),
getOpts().getDefaultExcludes(),
getOpts().getCaseSensitive(),
getOpts().getExcludeLocaleFilenames());
} catch (IOException e) {
throw Throwables.propagate(e);
}
}

abstract Collection<LocaleMapping> findLocales();
/**
* Try to find locales based on translation files found on local file
* system. It will be called for each source document in a loop.
*
* @return a list of locale mapping that potentially has translation files
* @param srcDocName
* source document name
*/
abstract Collection<LocaleMapping> findLocales(String srcDocName);

protected File getTransFile(LocaleMapping locale, String docName) {
File transFile = new TransFileResolver(getOpts()).getTransFile(
Expand All @@ -104,10 +130,10 @@ public Resource loadSrcDoc(File sourceDir, String docName)
}

@Override
public void visitTranslationResources(String docName, Resource srcDoc,
public void visitTranslationResources(String srcDocName, Resource srcDoc,
TranslationResourcesVisitor callback) throws IOException {
for (LocaleMapping locale : findLocales()) {
File transFile = getTransFile(locale, docName);
for (LocaleMapping locale : findLocales(srcDocName)) {
File transFile = getTransFile(locale, srcDocName);
if (transFile.canRead()) {
try (BufferedInputStream bis = new BufferedInputStream(
new FileInputStream(transFile))) {
Expand Down
Expand Up @@ -27,35 +27,53 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.zanata.client.commands.ConsoleInteractorImpl;
import org.zanata.client.commands.gettext.PublicanUtil;
import org.zanata.client.commands.TransFileResolver;
import org.zanata.client.commands.UnqualifiedSrcDocName;
import org.zanata.client.config.LocaleList;
import org.zanata.client.config.LocaleMapping;

import com.google.common.collect.Lists;

public class GettextDirStrategy extends AbstractGettextPushStrategy {
private static final Logger log = LoggerFactory
.getLogger(GettextDirStrategy.class);

@Override
List<LocaleMapping> findLocales() {
List<LocaleMapping> locales;
if (getOpts().getLocaleMapList() != null) {
locales =
PublicanUtil.findLocales(getOpts().getTransDir(), getOpts()
.getLocaleMapList());
if (locales.size() == 0) {
log.warn("'pushType' is set to '" + getOpts().getPushType()
+ "', but none of the configured locale "
+ "directories was found (check zanata.xml)");
}
} else {
locales = PublicanUtil.findLocales(getOpts().getTransDir());
if (locales.size() == 0) {
log.warn("'pushType' is set to '\" + getOpts().getPushType() + \"', but no locale directories were found");
List<LocaleMapping> findLocales(String srcDocName) {

List<LocaleMapping> localesFoundOnDisk = Lists.newArrayList();

LocaleList localeListInConfig = getOpts().getLocaleMapList();
if (localeListInConfig == null || localeListInConfig.isEmpty()) {
log.warn("No locale list in configuration (check zanata.xml)");
return localesFoundOnDisk;
}

for (LocaleMapping loc : localeListInConfig) {
if (hasTranslationFileForLocale(loc, srcDocName)) {
localesFoundOnDisk.add(loc);
} else {
log.info("'pushType' is set to '\" + getOpts().getPushType() + \"', but no locales specified in configuration: "
+ "importing " + locales.size() + " directories");
log.warn(
"configured locale {} not found; no translation file (local mapping {}) exist",
loc.getLocale(), loc.getLocalLocale());
}
}
return locales;

if (localesFoundOnDisk.size() == 0) {
log.warn(
"'pushType' is set to '{}', but none of the configured locale "
+ "files was found (check zanata.xml)", getOpts()
.getPushType());
}

return localesFoundOnDisk;
}

private boolean hasTranslationFileForLocale(LocaleMapping loc,
String srcDocName) {
File transFile = new TransFileResolver(getOpts()).getTransFile(
UnqualifiedSrcDocName.from(srcDocName), loc);
return transFile.exists();
}

@Override
Expand Down
@@ -1,54 +1,72 @@
package org.zanata.client.commands.push;

import static com.google.common.collect.Sets.newTreeSet;
import static org.apache.commons.io.FileUtils.listFiles;
import static org.apache.commons.io.FilenameUtils.removeExtension;

import java.io.File;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Set;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.zanata.client.commands.TransFileResolver;
import org.zanata.client.commands.UnqualifiedSrcDocName;
import org.zanata.client.config.LocaleList;
import org.zanata.client.config.LocaleMapping;

import com.google.common.base.Function;
import com.google.common.collect.Lists;

public class GettextPushStrategy extends AbstractGettextPushStrategy {
private static final Logger log = LoggerFactory
.getLogger(GettextPushStrategy.class);

@Override
List<LocaleMapping> findLocales() {
List<LocaleMapping> findLocales(String srcDocName) {
// find all .po basenames in this dir and subdirs
final LocaleList localeList = getOpts().getLocaleMapList();
Collection<File> files =
Collection<File> transFilesOnDisk =
listFiles(getOpts().getTransDir(), new String[] { "po" }, true);
Set<String> localeNames = newTreeSet();
for (File f : files) {
String localeName = removeExtension(f.getName());
localeNames.add(localeName);

final LocaleList localeListInConfig = getOpts().getLocaleMapList();

if (localeListInConfig == null || localeListInConfig.isEmpty()) {
log.warn("No locale list in configuration (check zanata.xml)");
return Collections.emptyList();
}
List<LocaleMapping> locales = Lists.newArrayList();
for (String localeName : localeNames) {
LocaleMapping localLocale;
if (localeList != null) {
localLocale = localeList.findByLocalLocaleOrJavaLocale(
localeName);
if (localLocale == null) {
log.warn(
"Skipping locale {}; no locale entry found in zanata.xml",
localeName);
continue;
}
} else {
localLocale = new LocaleMapping(localeName);
}
locales.add(localLocale);

final UnqualifiedSrcDocName unqualifiedSrcDocName =
UnqualifiedSrcDocName.from(srcDocName);
List<File> transFilesDestinations =
Lists.transform(localeListInConfig,
new LocaleMappingToTransFile(unqualifiedSrcDocName,
getOpts()));
// we remove all the ones that WILL be mapped and treated as
// translation files
transFilesOnDisk.removeAll(transFilesDestinations);
// for all remaining po files we give a warning
for (File transFile : transFilesOnDisk) {
log.warn(
"Skipping file {}; no locale entry found in zanata.xml",
transFile);
}
return locales;
return localeListInConfig;
}

private static class LocaleMappingToTransFile implements
Function<LocaleMapping, File> {
private final UnqualifiedSrcDocName unqualifiedSrcDocName;
private TransFileResolver transFileResolver;

public LocaleMappingToTransFile(
UnqualifiedSrcDocName unqualifiedSrcDocName, PushOptions opts) {
this.unqualifiedSrcDocName = unqualifiedSrcDocName;
transFileResolver = new TransFileResolver(opts);
}

@Override
public File apply(LocaleMapping localeMapping) {
return transFileResolver.getTransFile(unqualifiedSrcDocName,
localeMapping);
}
}
}
Expand Up @@ -3,6 +3,7 @@
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
Expand Down Expand Up @@ -34,8 +35,11 @@
import org.zanata.rest.dto.ProcessStatus;
import org.zanata.rest.dto.resource.Resource;
import org.zanata.rest.dto.resource.ResourceMeta;
import org.zanata.rest.dto.resource.TextFlowTarget;
import org.zanata.rest.dto.resource.TranslationsResource;

import com.google.common.base.Predicate;
import com.google.common.collect.Collections2;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.UniformInterfaceException;

Expand Down Expand Up @@ -392,7 +396,15 @@ private void pushCurrentModule() throws IOException, RuntimeException {
public void visit(LocaleMapping locale,
TranslationsResource targetDoc) {
debug(targetDoc);

stripUntranslatedEntriesIfMergeTypeIsNotImport(getOpts(),
targetDoc);
if (targetDoc.getTextFlowTargets()
.isEmpty()) {
log.debug(
"Skip translation file {}({}) since it has no translation in it",
localDocName, locale);
return;
}
pushTargetDocToServer(docUri, locale,
qualifiedDocName, targetDoc,
extensions);
Expand Down Expand Up @@ -431,6 +443,30 @@ public void visit(LocaleMapping locale,
deleteSourceDocsFromServer(obsoleteDocs);
}

private static void stripUntranslatedEntriesIfMergeTypeIsNotImport(
PushOptions opts, TranslationsResource translationResources) {
String mergeType = opts.getMergeType();
if (!MergeType.IMPORT.name().equalsIgnoreCase(mergeType)) {
List<TextFlowTarget> originalTargets =
translationResources.getTextFlowTargets();
Collection<TextFlowTarget> untranslatedEntries = Collections2
.filter(originalTargets,
new Predicate<TextFlowTarget>() {
@Override
public boolean apply(
TextFlowTarget input) {
return input == null ||
input.getState().isUntranslated();
}
});
log.debug(
"Remove {} untranslated entries from the payload since merge type is NOT import ({})",
untranslatedEntries.size(), mergeType);
translationResources.getTextFlowTargets()
.removeAll(untranslatedEntries);
}
}

/**
* Returns a list with all documents before fromDoc removed.
*
Expand Down
Expand Up @@ -23,6 +23,7 @@

import java.io.File;
import java.io.IOException;
import java.util.List;

import org.hamcrest.Matchers;
import org.junit.Before;
Expand All @@ -33,10 +34,10 @@
import org.zanata.client.config.LocaleList;
import org.zanata.client.config.LocaleMapping;
import com.google.common.base.Optional;
import com.google.common.collect.Sets;

import static com.google.common.collect.Lists.newArrayList;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.*;
import static org.zanata.client.TestUtils.createAndAddLocaleMapping;

public class GettextDirStrategyTest {
Expand Down Expand Up @@ -93,4 +94,30 @@ public void willWorkWithMappingRules() {
.getTransDir(), "zh-Hans/foo/message.po")));
}

// rhbz1174516
@Test
public void willWorkOnNonStandardTranslationDir() throws IOException {
opts.setPushType("trans");
opts.setFileMappingRules(newArrayList(
new FileMappingRule(null,
"man{path}/{locale_with_underscore}/{filename}.po")));
strategy.setPushOptions(opts);

LocaleMapping deMapping = createAndAddLocaleMapping("de",
Optional.<String>absent(), opts);
LocaleMapping zhMapping =
createAndAddLocaleMapping("zh-CN",
Optional.of("zh-Hans"),
opts);

// create some trans files
tempFileRule.createTransFileRelativeToTransDir("manfoo/de/message.po");
tempFileRule
.createTransFileRelativeToTransDir("manfoo/zh_Hans/message.po");

List<LocaleMapping> locales = strategy.findLocales("foo/message");

assertThat(locales, Matchers.containsInAnyOrder(deMapping, zhMapping));
}

}

0 comments on commit 1fbfeac

Please sign in to comment.