diff --git a/zanata-client-commands/src/main/java/org/zanata/client/commands/push/AbstractGettextPushStrategy.java b/zanata-client-commands/src/main/java/org/zanata/client/commands/push/AbstractGettextPushStrategy.java index 54b44b93..710ecc09 100644 --- a/zanata-client-commands/src/main/java/org/zanata/client/commands/push/AbstractGettextPushStrategy.java +++ b/zanata-client-commands/src/main/java/org/zanata/client/commands/push/AbstractGettextPushStrategy.java @@ -31,6 +31,7 @@ import java.util.HashSet; import java.util.Set; +import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import org.xml.sax.InputSource; import org.zanata.adapter.po.PoReader2; @@ -45,6 +46,7 @@ public abstract class AbstractGettextPushStrategy extends AbstractPushStrategy { private PoReader2 poReader = new PoReader2(); + protected Set localSrcDocNames; public AbstractGettextPushStrategy() { super(new StringSet("comment;gettext"), ".pot"); @@ -63,7 +65,10 @@ public Set findDocNames(File srcDir, ImmutableList includes, ImmutableList excludes, boolean useDefaultExclude, boolean caseSensitive, boolean excludeLocaleFilenames) throws IOException { - Set localDocNames = new HashSet(); + if (localSrcDocNames != null) { + return localSrcDocNames; + } + localSrcDocNames = new HashSet<>(); // populate localDocNames by looking in pot directory, ignore // excludeLocale option @@ -73,14 +78,35 @@ public Set findDocNames(File srcDir, ImmutableList includes, for (String potName : srcFiles) { String docName = removeExtension(potName); - localDocNames.add(docName); + localSrcDocNames.add(docName); } checkSrcFileNames(getOpts().getProjectType(), srcFiles, getOpts().isInteractiveMode()); - return localDocNames; + return localSrcDocNames; + } + + protected Set getSrcDocNames() { + try { + return findDocNames(getOpts().getSrcDir(), + getOpts().getIncludes(), + getOpts().getExcludes(), + getOpts().getDefaultExcludes(), + getOpts().getCaseSensitive(), + getOpts().getExcludeLocaleFilenames()); + } catch (IOException e) { + throw Throwables.propagate(e); + } } - abstract Collection findLocales(); + /** + * Try to find locales based on translation files found on local file + * system. It will be called for each source document in a loop. + * + * @return a list of locale mapping that potentially has translation files + * @param srcDocName + * source document name + */ + abstract Collection findLocales(String srcDocName); protected File getTransFile(LocaleMapping locale, String docName) { File transFile = new TransFileResolver(getOpts()).getTransFile( @@ -104,10 +130,10 @@ public Resource loadSrcDoc(File sourceDir, String docName) } @Override - public void visitTranslationResources(String docName, Resource srcDoc, + public void visitTranslationResources(String srcDocName, Resource srcDoc, TranslationResourcesVisitor callback) throws IOException { - for (LocaleMapping locale : findLocales()) { - File transFile = getTransFile(locale, docName); + for (LocaleMapping locale : findLocales(srcDocName)) { + File transFile = getTransFile(locale, srcDocName); if (transFile.canRead()) { try (BufferedInputStream bis = new BufferedInputStream( new FileInputStream(transFile))) { diff --git a/zanata-client-commands/src/main/java/org/zanata/client/commands/push/GettextDirStrategy.java b/zanata-client-commands/src/main/java/org/zanata/client/commands/push/GettextDirStrategy.java index 497ce386..7f0e843d 100644 --- a/zanata-client-commands/src/main/java/org/zanata/client/commands/push/GettextDirStrategy.java +++ b/zanata-client-commands/src/main/java/org/zanata/client/commands/push/GettextDirStrategy.java @@ -27,35 +27,53 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.zanata.client.commands.ConsoleInteractorImpl; -import org.zanata.client.commands.gettext.PublicanUtil; +import org.zanata.client.commands.TransFileResolver; +import org.zanata.client.commands.UnqualifiedSrcDocName; +import org.zanata.client.config.LocaleList; import org.zanata.client.config.LocaleMapping; +import com.google.common.collect.Lists; + public class GettextDirStrategy extends AbstractGettextPushStrategy { private static final Logger log = LoggerFactory .getLogger(GettextDirStrategy.class); @Override - List findLocales() { - List locales; - if (getOpts().getLocaleMapList() != null) { - locales = - PublicanUtil.findLocales(getOpts().getTransDir(), getOpts() - .getLocaleMapList()); - if (locales.size() == 0) { - log.warn("'pushType' is set to '" + getOpts().getPushType() - + "', but none of the configured locale " - + "directories was found (check zanata.xml)"); - } - } else { - locales = PublicanUtil.findLocales(getOpts().getTransDir()); - if (locales.size() == 0) { - log.warn("'pushType' is set to '\" + getOpts().getPushType() + \"', but no locale directories were found"); + List findLocales(String srcDocName) { + + List localesFoundOnDisk = Lists.newArrayList(); + + LocaleList localeListInConfig = getOpts().getLocaleMapList(); + if (localeListInConfig == null || localeListInConfig.isEmpty()) { + log.warn("No locale list in configuration (check zanata.xml)"); + return localesFoundOnDisk; + } + + for (LocaleMapping loc : localeListInConfig) { + if (hasTranslationFileForLocale(loc, srcDocName)) { + localesFoundOnDisk.add(loc); } else { - log.info("'pushType' is set to '\" + getOpts().getPushType() + \"', but no locales specified in configuration: " - + "importing " + locales.size() + " directories"); + log.warn( + "configured locale {} not found; no translation file (local mapping {}) exist", + loc.getLocale(), loc.getLocalLocale()); } } - return locales; + + if (localesFoundOnDisk.size() == 0) { + log.warn( + "'pushType' is set to '{}', but none of the configured locale " + + "files was found (check zanata.xml)", getOpts() + .getPushType()); + } + + return localesFoundOnDisk; + } + + private boolean hasTranslationFileForLocale(LocaleMapping loc, + String srcDocName) { + File transFile = new TransFileResolver(getOpts()).getTransFile( + UnqualifiedSrcDocName.from(srcDocName), loc); + return transFile.exists(); } @Override diff --git a/zanata-client-commands/src/main/java/org/zanata/client/commands/push/GettextPushStrategy.java b/zanata-client-commands/src/main/java/org/zanata/client/commands/push/GettextPushStrategy.java index caaaf163..b4ae0fba 100644 --- a/zanata-client-commands/src/main/java/org/zanata/client/commands/push/GettextPushStrategy.java +++ b/zanata-client-commands/src/main/java/org/zanata/client/commands/push/GettextPushStrategy.java @@ -1,19 +1,20 @@ package org.zanata.client.commands.push; -import static com.google.common.collect.Sets.newTreeSet; import static org.apache.commons.io.FileUtils.listFiles; -import static org.apache.commons.io.FilenameUtils.removeExtension; import java.io.File; import java.util.Collection; +import java.util.Collections; import java.util.List; -import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.zanata.client.commands.TransFileResolver; +import org.zanata.client.commands.UnqualifiedSrcDocName; import org.zanata.client.config.LocaleList; import org.zanata.client.config.LocaleMapping; +import com.google.common.base.Function; import com.google.common.collect.Lists; public class GettextPushStrategy extends AbstractGettextPushStrategy { @@ -21,34 +22,51 @@ public class GettextPushStrategy extends AbstractGettextPushStrategy { .getLogger(GettextPushStrategy.class); @Override - List findLocales() { + List findLocales(String srcDocName) { // find all .po basenames in this dir and subdirs - final LocaleList localeList = getOpts().getLocaleMapList(); - Collection files = + Collection transFilesOnDisk = listFiles(getOpts().getTransDir(), new String[] { "po" }, true); - Set localeNames = newTreeSet(); - for (File f : files) { - String localeName = removeExtension(f.getName()); - localeNames.add(localeName); + + final LocaleList localeListInConfig = getOpts().getLocaleMapList(); + + if (localeListInConfig == null || localeListInConfig.isEmpty()) { + log.warn("No locale list in configuration (check zanata.xml)"); + return Collections.emptyList(); } - List locales = Lists.newArrayList(); - for (String localeName : localeNames) { - LocaleMapping localLocale; - if (localeList != null) { - localLocale = localeList.findByLocalLocaleOrJavaLocale( - localeName); - if (localLocale == null) { - log.warn( - "Skipping locale {}; no locale entry found in zanata.xml", - localeName); - continue; - } - } else { - localLocale = new LocaleMapping(localeName); - } - locales.add(localLocale); + + final UnqualifiedSrcDocName unqualifiedSrcDocName = + UnqualifiedSrcDocName.from(srcDocName); + List transFilesDestinations = + Lists.transform(localeListInConfig, + new LocaleMappingToTransFile(unqualifiedSrcDocName, + getOpts())); + // we remove all the ones that WILL be mapped and treated as + // translation files + transFilesOnDisk.removeAll(transFilesDestinations); + // for all remaining po files we give a warning + for (File transFile : transFilesOnDisk) { + log.warn( + "Skipping file {}; no locale entry found in zanata.xml", + transFile); } - return locales; + return localeListInConfig; } + private static class LocaleMappingToTransFile implements + Function { + private final UnqualifiedSrcDocName unqualifiedSrcDocName; + private TransFileResolver transFileResolver; + + public LocaleMappingToTransFile( + UnqualifiedSrcDocName unqualifiedSrcDocName, PushOptions opts) { + this.unqualifiedSrcDocName = unqualifiedSrcDocName; + transFileResolver = new TransFileResolver(opts); + } + + @Override + public File apply(LocaleMapping localeMapping) { + return transFileResolver.getTransFile(unqualifiedSrcDocName, + localeMapping); + } + } } diff --git a/zanata-client-commands/src/main/java/org/zanata/client/commands/push/PushCommand.java b/zanata-client-commands/src/main/java/org/zanata/client/commands/push/PushCommand.java index 5b5facba..bbfc91fc 100644 --- a/zanata-client-commands/src/main/java/org/zanata/client/commands/push/PushCommand.java +++ b/zanata-client-commands/src/main/java/org/zanata/client/commands/push/PushCommand.java @@ -3,6 +3,7 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -34,8 +35,11 @@ import org.zanata.rest.dto.ProcessStatus; import org.zanata.rest.dto.resource.Resource; import org.zanata.rest.dto.resource.ResourceMeta; +import org.zanata.rest.dto.resource.TextFlowTarget; import org.zanata.rest.dto.resource.TranslationsResource; +import com.google.common.base.Predicate; +import com.google.common.collect.Collections2; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.UniformInterfaceException; @@ -392,7 +396,15 @@ private void pushCurrentModule() throws IOException, RuntimeException { public void visit(LocaleMapping locale, TranslationsResource targetDoc) { debug(targetDoc); - + stripUntranslatedEntriesIfMergeTypeIsNotImport(getOpts(), + targetDoc); + if (targetDoc.getTextFlowTargets() + .isEmpty()) { + log.debug( + "Skip translation file {}({}) since it has no translation in it", + localDocName, locale); + return; + } pushTargetDocToServer(docUri, locale, qualifiedDocName, targetDoc, extensions); @@ -431,6 +443,30 @@ public void visit(LocaleMapping locale, deleteSourceDocsFromServer(obsoleteDocs); } + private static void stripUntranslatedEntriesIfMergeTypeIsNotImport( + PushOptions opts, TranslationsResource translationResources) { + String mergeType = opts.getMergeType(); + if (!MergeType.IMPORT.name().equalsIgnoreCase(mergeType)) { + List originalTargets = + translationResources.getTextFlowTargets(); + Collection untranslatedEntries = Collections2 + .filter(originalTargets, + new Predicate() { + @Override + public boolean apply( + TextFlowTarget input) { + return input == null || + input.getState().isUntranslated(); + } + }); + log.debug( + "Remove {} untranslated entries from the payload since merge type is NOT import ({})", + untranslatedEntries.size(), mergeType); + translationResources.getTextFlowTargets() + .removeAll(untranslatedEntries); + } + } + /** * Returns a list with all documents before fromDoc removed. * diff --git a/zanata-client-commands/src/test/java/org/zanata/client/commands/push/GettextDirStrategyTest.java b/zanata-client-commands/src/test/java/org/zanata/client/commands/push/GettextDirStrategyTest.java index 5156d747..5312d072 100644 --- a/zanata-client-commands/src/test/java/org/zanata/client/commands/push/GettextDirStrategyTest.java +++ b/zanata-client-commands/src/test/java/org/zanata/client/commands/push/GettextDirStrategyTest.java @@ -23,6 +23,7 @@ import java.io.File; import java.io.IOException; +import java.util.List; import org.hamcrest.Matchers; import org.junit.Before; @@ -33,10 +34,10 @@ import org.zanata.client.config.LocaleList; import org.zanata.client.config.LocaleMapping; import com.google.common.base.Optional; +import com.google.common.collect.Sets; import static com.google.common.collect.Lists.newArrayList; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.*; import static org.zanata.client.TestUtils.createAndAddLocaleMapping; public class GettextDirStrategyTest { @@ -93,4 +94,30 @@ public void willWorkWithMappingRules() { .getTransDir(), "zh-Hans/foo/message.po"))); } + // rhbz1174516 + @Test + public void willWorkOnNonStandardTranslationDir() throws IOException { + opts.setPushType("trans"); + opts.setFileMappingRules(newArrayList( + new FileMappingRule(null, + "man{path}/{locale_with_underscore}/{filename}.po"))); + strategy.setPushOptions(opts); + + LocaleMapping deMapping = createAndAddLocaleMapping("de", + Optional.absent(), opts); + LocaleMapping zhMapping = + createAndAddLocaleMapping("zh-CN", + Optional.of("zh-Hans"), + opts); + + // create some trans files + tempFileRule.createTransFileRelativeToTransDir("manfoo/de/message.po"); + tempFileRule + .createTransFileRelativeToTransDir("manfoo/zh_Hans/message.po"); + + List locales = strategy.findLocales("foo/message"); + + assertThat(locales, Matchers.containsInAnyOrder(deMapping, zhMapping)); + } + }