diff --git a/zanata-cli/src/test/java/org/zanata/client/ClientToServerTest.java b/zanata-cli/src/test/java/org/zanata/client/ClientToServerTest.java index 4a5fbb0e..3b420f3a 100644 --- a/zanata-cli/src/test/java/org/zanata/client/ClientToServerTest.java +++ b/zanata-cli/src/test/java/org/zanata/client/ClientToServerTest.java @@ -47,7 +47,7 @@ public void testDisableSSLCertOption() { String project = "iok"; String version = "6.4"; client.processArgs(command, "--url", url, "--project", project, - "--project-version", version, "--disable-ssl-cert"); + "--project-version", version, "--disable-ssl-cert", "--details"); } @Test diff --git a/zanata-client-commands/src/main/java/org/zanata/client/commands/BasicOptions.java b/zanata-client-commands/src/main/java/org/zanata/client/commands/BasicOptions.java index 322bd346..3cf317ef 100644 --- a/zanata-client-commands/src/main/java/org/zanata/client/commands/BasicOptions.java +++ b/zanata-client-commands/src/main/java/org/zanata/client/commands/BasicOptions.java @@ -49,9 +49,9 @@ public interface BasicOptions { boolean isQuietSet(); - public boolean isInteractiveMode(); + boolean isInteractiveMode(); - public void setInteractiveMode(boolean interactiveMode); + void setInteractiveMode(boolean interactiveMode); /** * Used to generate the command line interface and its usage help. This name @@ -60,7 +60,7 @@ public interface BasicOptions { * * @return */ - public String getCommandName(); + String getCommandName(); /** * Used to generate CLI usage help. This description should preferably match @@ -68,9 +68,9 @@ public interface BasicOptions { * * @return */ - public String getCommandDescription(); + String getCommandDescription(); - public @Nonnull List getCommandHooks(); + @Nonnull List getCommandHooks(); - public void setCommandHooks(@Nonnull List commandHooks); + void setCommandHooks(@Nonnull List commandHooks); } diff --git a/zanata-client-commands/src/main/java/org/zanata/client/commands/ConfigurableOptions.java b/zanata-client-commands/src/main/java/org/zanata/client/commands/ConfigurableOptions.java index af94e989..acb688fb 100644 --- a/zanata-client-commands/src/main/java/org/zanata/client/commands/ConfigurableOptions.java +++ b/zanata-client-commands/src/main/java/org/zanata/client/commands/ConfigurableOptions.java @@ -18,37 +18,37 @@ public interface ConfigurableOptions extends BasicOptions { /** * API key for accessing the REST API. Defaults to the value in zanata.ini. */ - public String getKey(); + String getKey(); - public void setKey(String key); + void setKey(String key); /** * Base URL for the server. Defaults to the value in zanata.xml. */ - public URL getUrl(); + URL getUrl(); - public void setUrl(URL url); + void setUrl(URL url); /** * Client configuration file. */ - public File getUserConfig(); + File getUserConfig(); - public void setUserConfig(File userConfig); + void setUserConfig(File userConfig); /** * Username for accessing the REST API. Defaults to the value in zanata.ini. */ - public String getUsername(); + String getUsername(); - public void setUsername(String username); + void setUsername(String username); /** * Enable HTTP message logging. */ - public boolean getLogHttp(); + boolean getLogHttp(); - public void setLogHttp(boolean traceLogging); + void setLogHttp(boolean traceLogging); /** * Disable SSL certificate verification when connecting to Zanata host by @@ -56,13 +56,12 @@ public interface ConfigurableOptions extends BasicOptions { */ boolean isDisableSSLCert(); - public - void setDisableSSLCert(boolean disableSSLCert); + void setDisableSSLCert(boolean disableSSLCert); /** * Use to disable check for presence of username and API key before running command. * * @return true if this command should fail when username or API key is absent. */ - public boolean isAuthRequired(); + boolean isAuthRequired(); } diff --git a/zanata-client-commands/src/main/java/org/zanata/client/commands/PushPullCommand.java b/zanata-client-commands/src/main/java/org/zanata/client/commands/PushPullCommand.java index eff1e549..9a425fd4 100644 --- a/zanata-client-commands/src/main/java/org/zanata/client/commands/PushPullCommand.java +++ b/zanata-client-commands/src/main/java/org/zanata/client/commands/PushPullCommand.java @@ -28,24 +28,33 @@ import java.io.StringWriter; import java.util.ArrayList; import java.util.List; +import java.util.Map; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.zanata.client.commands.pull.PullOptions; import org.zanata.client.config.LocaleList; import org.zanata.client.config.LocaleMapping; import org.zanata.client.etag.ETagCache; import org.zanata.client.etag.ETagCacheReaderWriter; import org.zanata.client.exceptions.ConfigException; +import org.zanata.common.LocaleId; import org.zanata.rest.client.RestClientFactory; import org.zanata.rest.client.SourceDocResourceClient; +import org.zanata.rest.client.StatisticsResourceClient; import org.zanata.rest.client.TransDocResourceClient; import org.zanata.rest.dto.resource.Resource; import org.zanata.rest.dto.resource.ResourceMeta; import org.zanata.rest.dto.resource.TranslationsResource; +import org.zanata.rest.dto.stats.ContainerTranslationStatistics; +import org.zanata.rest.dto.stats.TranslationStatistics; import org.zanata.util.PathUtil; import static org.zanata.client.commands.ConsoleInteractor.DisplayMode.Question; @@ -67,6 +76,7 @@ public abstract class PushPullCommand extends private String modulePrefix; protected SourceDocResourceClient sourceDocResourceClient; protected TransDocResourceClient transDocResourceClient; + protected final StatisticsResourceClient statsClient; public PushPullCommand(O opts, RestClientFactory clientFactory) { super(opts, clientFactory); @@ -80,6 +90,7 @@ public PushPullCommand(O opts, RestClientFactory clientFactory) { transDocResourceClient = getClientFactory().getTransDocResourceClient(opts.getProj(), opts.getProjectVersion()); + statsClient = getClientFactory().getStatisticsClient(); } public PushPullCommand(O opts) { @@ -241,4 +252,125 @@ protected void storeETagCache() { } } + protected Map> getDocsTranslatedPercent( + LocaleList locales) { + ContainerTranslationStatistics statistics = + getDetailStatisticsForProjectVersion(locales); + List statsPerDoc = + statistics.getDetailedStats(); + ImmutableMap.Builder> docIdToStatsBuilder = + ImmutableMap.builder(); + for (ContainerTranslationStatistics docStats : statsPerDoc) { + String docId = docStats.getId(); + List statsPerLocale = docStats.getStats(); + ImmutableMap.Builder localeToStatsBuilder = + ImmutableMap.builder(); + + for (TranslationStatistics statsForSingleLocale : statsPerLocale) { + // TODO server statistics API should return locale with alias + TranslatedPercent translatedPercent = + new TranslatedPercent(statsForSingleLocale.getTotal(), + statsForSingleLocale.getTranslatedOnly(), + statsForSingleLocale.getApproved()); + + localeToStatsBuilder.put( + new LocaleId(statsForSingleLocale.getLocale()), + translatedPercent); + } + Map localeStats = + localeToStatsBuilder.build(); + docIdToStatsBuilder.put(docId, localeStats); + } + return docIdToStatsBuilder.build(); + } + + @VisibleForTesting + protected ContainerTranslationStatistics getDetailStatisticsForProjectVersion( + LocaleList locales) { + String[] localesOnServer = new String[locales.size()]; + for (int i = 0; i < locales.size(); i++) { + localesOnServer[i] = locales.get(i).getLocale(); + } + return statsClient + .getStatistics(getOpts().getProj(), + getOpts().getProjectVersion(), true, false, localesOnServer); + } + + /** + * this stats map will have docId as key, the value is another map with + * localeId as key and translated percent as value. + * It's optional if we require statistics to determine which file to pull. + * In cases where statistics is not required, + * i.e. pull source only or minimum percent is set to 0, this will be + * Optional.absence(). + * + * @param pullTarget whether we need to pull translation target + * @param locales + * @return either detailed document statistics or optional.absence() + */ + protected Optional>> prepareStatsIfApplicable( + boolean pullTarget, LocaleList locales) { + + Optional>> optionalStats = + Optional.absent(); + if (needToGetStatistics(pullTarget)) { + optionalStats = Optional.of(getDocsTranslatedPercent(locales)); + } + return optionalStats; + } + + protected boolean needToGetStatistics(boolean pullTarget) { + return pullTarget && getOpts() instanceof PullOptions && + ((PullOptions) getOpts()).getMinDocPercent() > 0; + } + + protected boolean shouldPullThisLocale( + Optional>> optionalStats, + String localDocName, LocaleId serverLocale) { + int minDocPercent = ((PullOptions) getOpts()).getMinDocPercent(); + if (log.isDebugEnabled() && optionalStats.isPresent()) { + log.debug("{} for locale {} is translated {}%", localDocName, + serverLocale, optionalStats.get() + .get(localDocName).get(serverLocale) + .getTranslatedPercent()); + } + return !optionalStats.isPresent() + || optionalStats.get().get(localDocName).get(serverLocale) + .isAboveThreshold(minDocPercent); + } + + protected static class TranslatedPercent { + private final double translatedPercent; + private final long total; + private final long translated; + private final long approved; + + public TranslatedPercent(long total, long translated, long approved) { + this.total = total; + this.translated = translated; + this.approved = approved; + if (total == 0) { + // in some case the document has no content, we want to pull the + // translation file regardless + translatedPercent = 100; + } else { + translatedPercent = (translated + approved) * 100.0 / total; + + } + } + + public boolean isAboveThreshold(int minimumPercent) { + // if minimum percent is 100, we will compare exact number so that + // rounding issue won't affect the result + if (minimumPercent == 100) { + return total == translated + approved; + } else { + return translatedPercent >= minimumPercent; + } + } + + public double getTranslatedPercent() { + return translatedPercent; + } + } } diff --git a/zanata-client-commands/src/main/java/org/zanata/client/commands/PushPullOptions.java b/zanata-client-commands/src/main/java/org/zanata/client/commands/PushPullOptions.java index dbd8d417..bd0763ab 100644 --- a/zanata-client-commands/src/main/java/org/zanata/client/commands/PushPullOptions.java +++ b/zanata-client-commands/src/main/java/org/zanata/client/commands/PushPullOptions.java @@ -55,7 +55,7 @@ public interface PushPullOptions extends ConfigurableProjectOptions { File getTransDir(); - public String getFromDoc(); + String getFromDoc(); /** * This name should represent the exact parameter as it would be entered on @@ -63,7 +63,7 @@ public interface PushPullOptions extends ConfigurableProjectOptions { * parameter to the argument. This is so that the argument can be appended * directly to the parameter name. */ - public String buildFromDocArgument(String argValue); + String buildFromDocArgument(String argValue); boolean getEnableModules(); diff --git a/zanata-client-commands/src/main/java/org/zanata/client/commands/pull/PullCommand.java b/zanata-client-commands/src/main/java/org/zanata/client/commands/pull/PullCommand.java index 9bceaa14..39215e6f 100644 --- a/zanata-client-commands/src/main/java/org/zanata/client/commands/pull/PullCommand.java +++ b/zanata-client-commands/src/main/java/org/zanata/client/commands/pull/PullCommand.java @@ -30,6 +30,9 @@ import org.zanata.rest.dto.resource.TranslationsResource; import org.zanata.util.HashUtil; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.collect.Lists; import com.sun.jersey.api.client.ClientResponse; /** @@ -123,12 +126,16 @@ public static void logOptions(Logger logger, PullOptions opts) { logger.info("Pulling target documents (translations) only"); logger.info("Target-language base directory (translations): {}", opts.getTransDir()); + logger.info("Minimum accepted translation percentage (message based): {}%", + opts.getMinDocPercent()); } else { logger.info("Pulling source and target (translation) documents"); logger.info("Source-language directory (originals): {}", opts.getSrcDir()); logger.info("Target-language base directory (translations): {}", opts.getTransDir()); + logger.info("Minimum accepted translation percentage (message based): {}%", + opts.getMinDocPercent()); } } @@ -137,10 +144,10 @@ public void run() throws Exception { logOptions(); LocaleList locales = getOpts().getLocaleMapList(); - if (locales == null && (getOpts().getPullType() != PushPullType.Source)) + if (locales == null && (getOpts().getPullType() != PushPullType.Source)) { throw new ConfigException("no locales specified"); - PullStrategy strat = createStrategy( - getOpts()); + } + PullStrategy strat = createStrategy(getOpts()); if (strat.isTransOnly() && getOpts().getPullType() == PushPullType.Source) { @@ -192,18 +199,26 @@ && getOpts().getPullType() == PushPullType.Source) { pullSrc = false; } + if (needToGetStatistics(pullTarget)) { + log.info("Setting minimum document completion percentage may potentially increase the processing time."); + } + if (pullSrc) { log.warn("Pull Type set to '" + pullType + "': existing source-language files may be overwritten/deleted"); confirmWithUser("This will overwrite/delete any existing documents and translations in the above directories.\n"); } else { - confirmWithUser("This will overwrite/delete any existing translations in the above directory.\n"); + confirmWithUser( + "This will overwrite/delete any existing translations in the above directory.\n"); } if (getOpts().getPurgeCache()) { eTagCache.clear(); } + Optional>> optionalStats = + prepareStatsIfApplicable(pullTarget, locales); + for (String qualifiedDocName : docsToPull) { try { @@ -223,83 +238,25 @@ && getOpts().getPullType() == PushPullType.Source) { } if (pullTarget) { + List skippedLocales = Lists.newArrayList(); for (LocaleMapping locMapping : locales) { LocaleId locale = new LocaleId(locMapping.getLocale()); - String eTag = null; File transFile = strat.getTransFileToWrite(localDocName, locMapping); - ETagCacheEntry eTagCacheEntry = - eTagCache.findEntry(localDocName, - locale.getId()); - - if (getOpts().getUseCache() && eTagCacheEntry != null) { - // Check the last updated date on the file matches - // what's in the cache - // only then use the cached ETag - if (transFile.exists() - && Long.toString(transFile.lastModified()) - .equals(eTagCacheEntry - .getLocalFileTime())) { - eTag = eTagCacheEntry.getServerETag(); - } - } - ClientResponse transResponse = - transDocResourceClient.getTranslations(docUri, - locale, strat.getExtensions(), - createSkeletons, eTag); - - // ignore 404 (no translation yet for specified - // document) - if (transResponse.getClientResponseStatus() == ClientResponse.Status.NOT_FOUND) { - if (!createSkeletons) { - log.info( - "No translations found in locale {} for document {}", - locale, localDocName); - } else { - // Write the skeleton - writeTargetDoc(strat, localDocName, locMapping, - doc, null, - transResponse.getHeaders() - .getFirst(HttpHeaders.ETAG)); - } - } else if (transResponse.getClientResponseStatus() == ClientResponse.Status.NOT_MODIFIED) { - // 304 NOT MODIFIED (the document can stay the same) - log.info( - "No changes in translations for locale {} and document {}", - locale, localDocName); - - // Check the file's MD5 matches what's stored in the - // cache. If not, it needs to be fetched again (with - // no etag) - String fileChecksum = - HashUtil.getMD5Checksum(transFile); - if (!fileChecksum.equals(eTagCacheEntry - .getLocalFileMD5())) { - transResponse = - transDocResourceClient.getTranslations( - docUri, locale, - strat.getExtensions(), - createSkeletons, null); - ClientUtil.checkResult(transResponse); - // rewrite the target document - writeTargetDoc(strat, localDocName, locMapping, - doc, transResponse.getEntity(TranslationsResource.class), - transResponse.getHeaders() - .getFirst(HttpHeaders.ETAG)); - } + if (shouldPullThisLocale(optionalStats, localDocName, locale)) { + pullDocForLocale(strat, doc, localDocName, docUri, + createSkeletons, locMapping, transFile); } else { - ClientUtil.checkResult(transResponse); - TranslationsResource targetDoc = - transResponse.getEntity(TranslationsResource.class); - - // Write the target document - writeTargetDoc(strat, localDocName, locMapping, - doc, targetDoc, - transResponse.getHeaders() - .getFirst(HttpHeaders.ETAG)); + skippedLocales.add(locale); } + + } + if (!skippedLocales.isEmpty()) { + log.info( + "Translation file for document {} for locales {} are skipped due to insufficient completed percentage", + localDocName, skippedLocales); } // write the cache @@ -330,6 +287,86 @@ && getOpts().getPullType() == PushPullType.Source) { } + @VisibleForTesting + protected void pullDocForLocale(PullStrategy strat, Resource doc, + String localDocName, String docUri, boolean createSkeletons, + LocaleMapping locMapping, + File transFile) throws IOException { + LocaleId locale = new LocaleId(locMapping.getLocale()); + String eTag = null; + ETagCacheEntry eTagCacheEntry = + eTagCache.findEntry(localDocName, + locale.getId()); + + if (getOpts().getUseCache() && eTagCacheEntry != null) { + // Check the last updated date on the file matches + // what's in the cache + // only then use the cached ETag + if (transFile.exists() + && Long.toString(transFile.lastModified()) + .equals(eTagCacheEntry + .getLocalFileTime())) { + eTag = eTagCacheEntry.getServerETag(); + } + } + + ClientResponse transResponse = + transDocResourceClient.getTranslations(docUri, + locale, strat.getExtensions(), + createSkeletons, eTag); + + // ignore 404 (no translation yet for specified + // document) + if (transResponse.getClientResponseStatus() == ClientResponse.Status.NOT_FOUND) { + if (!createSkeletons) { + log.info( + "No translations found in locale {} for document {}", + locale, localDocName); + } else { + // Write the skeleton + writeTargetDoc(strat, localDocName, locMapping, + doc, null, + transResponse.getHeaders() + .getFirst(HttpHeaders.ETAG)); + } + } else if (transResponse.getClientResponseStatus() == ClientResponse.Status.NOT_MODIFIED) { + // 304 NOT MODIFIED (the document can stay the same) + log.info( + "No changes in translations for locale {} and document {}", + locale, localDocName); + + // Check the file's MD5 matches what's stored in the + // cache. If not, it needs to be fetched again (with + // no etag) + String fileChecksum = + HashUtil.getMD5Checksum(transFile); + if (!fileChecksum.equals(eTagCacheEntry + .getLocalFileMD5())) { + transResponse = + transDocResourceClient.getTranslations( + docUri, locale, + strat.getExtensions(), + createSkeletons, null); + ClientUtil.checkResult(transResponse); + // rewrite the target document + writeTargetDoc(strat, localDocName, locMapping, + doc, transResponse.getEntity(TranslationsResource.class), + transResponse.getHeaders() + .getFirst(HttpHeaders.ETAG)); + } + } else { + ClientUtil.checkResult(transResponse); + TranslationsResource targetDoc = + transResponse.getEntity(TranslationsResource.class); + + // Write the target document + writeTargetDoc(strat, localDocName, locMapping, + doc, targetDoc, + transResponse.getHeaders() + .getFirst(HttpHeaders.ETAG)); + } + } + /** * Returns a list with all documents before fromDoc removed. * diff --git a/zanata-client-commands/src/main/java/org/zanata/client/commands/pull/PullOptions.java b/zanata-client-commands/src/main/java/org/zanata/client/commands/pull/PullOptions.java index 2494c143..eb8ea04d 100644 --- a/zanata-client-commands/src/main/java/org/zanata/client/commands/pull/PullOptions.java +++ b/zanata-client-commands/src/main/java/org/zanata/client/commands/pull/PullOptions.java @@ -43,4 +43,6 @@ public interface PullOptions extends PushPullOptions { boolean getUseCache(); boolean isContinueAfterError(); + + int getMinDocPercent(); } diff --git a/zanata-client-commands/src/main/java/org/zanata/client/commands/pull/PullOptionsImpl.java b/zanata-client-commands/src/main/java/org/zanata/client/commands/pull/PullOptionsImpl.java index c77775f4..e1703a1c 100644 --- a/zanata-client-commands/src/main/java/org/zanata/client/commands/pull/PullOptionsImpl.java +++ b/zanata-client-commands/src/main/java/org/zanata/client/commands/pull/PullOptionsImpl.java @@ -26,6 +26,7 @@ import org.zanata.client.commands.BooleanValueHandler; import org.zanata.client.commands.ZanataCommand; import org.zanata.client.commands.PushPullType; +import com.google.common.base.Preconditions; /** * @author Sean Flanigan private boolean useCache = DEFAULT_USE_CACHE; private boolean purgeCache = DEFAULT_PURGE_CACHE; private boolean continueAfterError = DEFAULT_CONTINUE_AFTER_ERROR; + private int minDocPercent = 0; @Override public ZanataCommand initCommand() { @@ -165,6 +167,21 @@ public boolean isContinueAfterError() { return continueAfterError; } + @Override + public int getMinDocPercent() { + return this.minDocPercent; + } + + @Option(name = "--min-doc-percent", metaVar = "PERCENT", + usage = "Accepts integer from 0 to 100. Only pull translation documents which are at least PERCENT % (message based) completed.\n" + + "Please note specifying this option may cause longer time to pull for a large project") + public void setMinDocPercent(int minDocPercent) { + Preconditions + .checkArgument(minDocPercent >= 0 && minDocPercent <= 100, + "--min-doc-percent should be an integer from 0 to 100"); + this.minDocPercent = minDocPercent; + } + @Override public boolean isAuthRequired() { return false; diff --git a/zanata-client-commands/src/main/java/org/zanata/client/commands/pull/RawPullCommand.java b/zanata-client-commands/src/main/java/org/zanata/client/commands/pull/RawPullCommand.java index 4bec6af9..6df41791 100644 --- a/zanata-client-commands/src/main/java/org/zanata/client/commands/pull/RawPullCommand.java +++ b/zanata-client-commands/src/main/java/org/zanata/client/commands/pull/RawPullCommand.java @@ -23,9 +23,11 @@ import java.io.IOException; import java.io.InputStream; import java.util.List; +import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; +import com.google.common.collect.Lists; import org.apache.commons.io.FilenameUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,7 +36,6 @@ import org.zanata.client.config.LocaleList; import org.zanata.client.config.LocaleMapping; import org.zanata.client.exceptions.ConfigException; -import org.zanata.common.DocumentType; import org.zanata.common.LocaleId; import org.zanata.rest.client.ClientUtil; import org.zanata.rest.client.FileResourceClient; @@ -45,8 +46,6 @@ import com.google.common.base.Optional; import com.sun.jersey.api.client.ClientResponse; -import javax.ws.rs.core.HttpHeaders; - /** * * @author David Mason, >> optionalStats = + prepareStatsIfApplicable(pullTarget, locales); + for (String qualifiedDocName : docsToPull) { // TODO add filtering by file type? e.g. pull all dtd documents // only. @@ -188,40 +194,23 @@ public void run() throws IOException { FileResource.FILETYPE_TRANSLATED_APPROVED; } + List skippedLocales = Lists.newArrayList(); for (LocaleMapping locMapping : locales) { LocaleId locale = new LocaleId(locMapping.getLocale()); - ClientResponse response = - fileResourceClient.downloadTranslationFile(getOpts() - .getProj(), getOpts() - .getProjectVersion(), locale.getId(), - fileExtension, qualifiedDocName); - if (response.getClientResponseStatus() == ClientResponse.Status.NOT_FOUND) { - log.info( - "No translation document file found in locale {} for document [{}]", - locale, qualifiedDocName); + + if (shouldPullThisLocale(optionalStats, localDocName, locale)) { + pullDocForLocale(strat, qualifiedDocName, localDocName, + fileExtension, + locMapping, locale); } else { - ClientUtil.checkResult(response); - InputStream transDoc = - (InputStream) response - .getEntity(InputStream.class); - if (transDoc != null) { - try { - String fileName = - ClientUtil.getFileNameFromHeader( - response.getHeaders()); - String targetFileExt = FilenameUtils - .getExtension(fileName); - - Optional translationFileExtension = - Optional.fromNullable(targetFileExt); - - strat.writeTransFile(localDocName, - locMapping, transDoc, translationFileExtension); - } finally { - transDoc.close(); - } - } + skippedLocales.add(locale); } + + } + if (!skippedLocales.isEmpty()) { + log.info( + "Translation file for document {} for locales {} are skipped due to insufficient completed percentage", + localDocName, skippedLocales); } } } catch (IOException | RuntimeException e) { @@ -233,4 +222,39 @@ public void run() throws IOException { } } } + + private void pullDocForLocale(RawPullStrategy strat, + String qualifiedDocName, String localDocName, String fileExtension, + LocaleMapping locMapping, LocaleId locale) throws IOException { + ClientResponse response = + fileResourceClient.downloadTranslationFile(getOpts() + .getProj(), getOpts() + .getProjectVersion(), locale.getId(), + fileExtension, qualifiedDocName); + if (response.getClientResponseStatus() == ClientResponse.Status.NOT_FOUND) { + log.info( + "No translation document file found in locale {} for document [{}]", + locale, qualifiedDocName); + } else { + ClientUtil.checkResult(response); + InputStream transDoc = response.getEntity(InputStream.class); + if (transDoc != null) { + try { + String fileName = + ClientUtil.getFileNameFromHeader( + response.getHeaders()); + String targetFileExt = FilenameUtils + .getExtension(fileName); + + Optional translationFileExtension = + Optional.fromNullable(targetFileExt); + + strat.writeTransFile(localDocName, + locMapping, transDoc, translationFileExtension); + } finally { + transDoc.close(); + } + } + } + } } diff --git a/zanata-client-commands/src/main/java/org/zanata/client/commands/push/PushCommand.java b/zanata-client-commands/src/main/java/org/zanata/client/commands/push/PushCommand.java index 77697154..4bd49df2 100644 --- a/zanata-client-commands/src/main/java/org/zanata/client/commands/push/PushCommand.java +++ b/zanata-client-commands/src/main/java/org/zanata/client/commands/push/PushCommand.java @@ -63,7 +63,7 @@ public class PushCommand extends PushPullCommand { private CopyTransClient copyTransClient; private AsyncProcessClient asyncProcessClient; - public static interface TranslationResourcesVisitor { + public interface TranslationResourcesVisitor { void visit(LocaleMapping locale, TranslationsResource targetDoc); } diff --git a/zanata-client-commands/src/test/java/org/zanata/client/commands/pull/PullCommandTest.java b/zanata-client-commands/src/test/java/org/zanata/client/commands/pull/PullCommandTest.java new file mode 100644 index 00000000..4f18e919 --- /dev/null +++ b/zanata-client-commands/src/test/java/org/zanata/client/commands/pull/PullCommandTest.java @@ -0,0 +1,286 @@ +/* + * Copyright 2015, Red Hat, Inc. and individual contributors + * as indicated by the @author tags. See the copyright.txt file in the + * distribution for a full listing of individual contributors. + * + * This is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ + +package org.zanata.client.commands.pull; + +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.zanata.client.config.LocaleList; +import org.zanata.client.config.LocaleMapping; +import org.zanata.common.LocaleId; +import org.zanata.common.ProjectType; +import org.zanata.common.TransUnitCount; +import org.zanata.rest.StringSet; +import org.zanata.rest.client.RestClientFactory; +import org.zanata.rest.client.SourceDocResourceClient; +import org.zanata.rest.client.StatisticsResourceClient; +import org.zanata.rest.client.TransDocResourceClient; +import org.zanata.rest.dto.resource.Resource; +import org.zanata.rest.dto.stats.ContainerTranslationStatistics; +import org.zanata.rest.dto.stats.TranslationStatistics; + +import com.google.common.collect.Lists; + +public class PullCommandTest { + public static final StringSet EXTENSIONS = new StringSet("comment"); + @Mock + private RestClientFactory restClientFactory; + private PullOptionsImpl opts; + private final String projectSlug = "project"; + private final String versionSlug = "master"; + @Mock + private SourceDocResourceClient sourceClient; + @Mock + private TransDocResourceClient transClient; + @Mock + private StatisticsResourceClient statsClient; + private LocaleList locales; + private PullCommand pullCommand; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + opts = new PullOptionsImpl(); + opts.setProj(projectSlug); + opts.setProjectVersion(versionSlug); + opts.setProjectType(ProjectType.Properties.name().toLowerCase()); + opts.setBatchMode(true); + + when( + restClientFactory.getSourceDocResourceClient(projectSlug, + versionSlug)).thenReturn( + sourceClient); + when(restClientFactory.getTransDocResourceClient(projectSlug, + versionSlug)).thenReturn(transClient); + when(restClientFactory.getStatisticsClient()).thenReturn(statsClient); + + locales = new LocaleList(); + opts.setLocaleMapList(locales); + pullCommand = new PullCommand(opts, restClientFactory); + } + + @Test + public void pullSourceOnlyWillIgnoreMinimumPercent() throws Exception { + locales.add(new LocaleMapping("zh")); + locales.add(new LocaleMapping("de")); + opts.setDryRun(true); + // Given: pull-type is source only and minimum-doc-percent is set to 80 + opts.setPullType("source"); + opts.setMinDocPercent(80); + when(sourceClient + .getResource("file1", EXTENSIONS)).thenReturn( + new Resource()); + pullCommand = new PullCommand(opts, restClientFactory) { + @Override + protected List + getQualifiedDocNamesForCurrentModuleFromServer() { + return Lists.newArrayList("file1"); + } + }; + + // When: + pullCommand.run(); + + // Then: + verifyZeroInteractions(statsClient, transClient); + } + + @Test + public void pullTransOnlyWillIgnoreMinimumPercentIfItIsZero() + throws Exception { + locales.add(new LocaleMapping("zh")); + locales.add(new LocaleMapping("de")); + opts.setDryRun(true); + // Given: pull-type is trans only and minimum-doc-percent is set to 0 + opts.setPullType("trans"); + opts.setMinDocPercent(0); + + pullCommand = new PullCommand(opts, restClientFactory) { + @Override + protected List + getQualifiedDocNamesForCurrentModuleFromServer() { + return Lists.newArrayList("file1"); + } + + @Override + protected void pullDocForLocale(PullStrategy strat, Resource doc, + String localDocName, String docUri, + boolean createSkeletons, + LocaleMapping locMapping, File transFile) + throws IOException { + // pretend we are pulling + transClient.getTranslations(docUri, + new LocaleId(locMapping.getLocale()), EXTENSIONS, + createSkeletons, null); + } + }; + + // When: + pullCommand.run(); + + // Then: + verifyZeroInteractions(statsClient); + verify(transClient).getTranslations("file1", new LocaleId("zh"), + EXTENSIONS, false, + null); + verify(transClient).getTranslations("file1", new LocaleId("de"), + EXTENSIONS, false, + null); + } + + @Test + public void pullTransOnlyWillUseMinimumPercentIfItIsNotZero() + throws Exception { + locales.add(new LocaleMapping("zh")); + locales.add(new LocaleMapping("de")); + opts.setDryRun(true); + // Given: pull-type is trans only and minimum-doc-percent is set to 80 + opts.setPullType("trans"); + opts.setMinDocPercent(80); + + ContainerTranslationStatistics statistics = + new ContainerTranslationStatistics(); + ContainerTranslationStatistics docStats = + new ContainerTranslationStatistics(); + docStats.setId("file1"); + statistics.addDetailedStats(docStats); + // zh has 100 approved + TranslationStatistics zhLocaleStats = + new TranslationStatistics(new TransUnitCount(100, 0, 0, 0, 0), + "zh"); + // de has 39 approved, 21 fuzzy and 40 translated so 79% translated + TranslationStatistics deLocaleStats = + new TranslationStatistics(new TransUnitCount(39, 21, 0, 40, 0), + "de"); + docStats.addStats(zhLocaleStats); + docStats.addStats(deLocaleStats); + + when(statsClient + .getStatistics(projectSlug, versionSlug, true, false, new String[] {"zh", "de"})) + .thenReturn(statistics); + + pullCommand = new PullCommand(opts, restClientFactory) { + @Override + protected List + getQualifiedDocNamesForCurrentModuleFromServer() { + return Lists.newArrayList("file1"); + } + + @Override + protected void pullDocForLocale(PullStrategy strat, Resource doc, + String localDocName, String docUri, + boolean createSkeletons, + LocaleMapping locMapping, File transFile) + throws IOException { + // pretend we are pulling + transClient.getTranslations(docUri, + new LocaleId(locMapping.getLocale()), EXTENSIONS, + createSkeletons, null); + } + }; + + // When: + pullCommand.run(); + + // Then: translation for "de" will not be pulled + verify(statsClient).getStatistics(projectSlug, versionSlug, true, + false, new String[] {"zh", "de"}); + verify(transClient).getTranslations("file1", new LocaleId("zh"), + EXTENSIONS, false, + null); + verifyNoMoreInteractions(transClient); + } + + @Test + public void whenMinimumPercentIsSetTo100ItWillUseTotalNumber() + throws Exception { + locales.add(new LocaleMapping("zh")); + locales.add(new LocaleMapping("de")); + opts.setDryRun(true); + // Given: pull-type is trans only and minimum-doc-percent is set to 100 + opts.setPullType("trans"); + opts.setMinDocPercent(100); + + ContainerTranslationStatistics statistics = + new ContainerTranslationStatistics(); + ContainerTranslationStatistics docStats = + new ContainerTranslationStatistics(); + docStats.setId("file1"); + statistics.addDetailedStats(docStats); + // zh has 100000 approved + TranslationStatistics zhLocaleStats = + new TranslationStatistics(new TransUnitCount(100000, 0, 0, 0, 0), + "zh"); + // de has 99999 approved, 1 untranslated so 99.999% translated + TranslationStatistics deLocaleStats = + new TranslationStatistics(new TransUnitCount(99999, 0, 1, 0, 0), + "de"); + docStats.addStats(zhLocaleStats); + docStats.addStats(deLocaleStats); + + when(statsClient + .getStatistics(projectSlug, versionSlug, true, false, new String[] {"zh", "de"})) + .thenReturn(statistics); + + pullCommand = new PullCommand(opts, restClientFactory) { + @Override + protected List + getQualifiedDocNamesForCurrentModuleFromServer() { + return Lists.newArrayList("file1"); + } + + @Override + protected void pullDocForLocale(PullStrategy strat, Resource doc, + String localDocName, String docUri, + boolean createSkeletons, + LocaleMapping locMapping, File transFile) + throws IOException { + // pretend we are pulling + transClient.getTranslations(docUri, + new LocaleId(locMapping.getLocale()), EXTENSIONS, + createSkeletons, null); + } + }; + + // When: + pullCommand.run(); + + // Then: translation for "de" will not be pulled + verify(statsClient).getStatistics(projectSlug, versionSlug, true, + false, new String[] {"zh", "de"}); + verify(transClient).getTranslations("file1", new LocaleId("zh"), + EXTENSIONS, false, + null); + verifyNoMoreInteractions(transClient); + } + +} diff --git a/zanata-maven-plugin/src/main/java/org/zanata/maven/AbstractPullMojo.java b/zanata-maven-plugin/src/main/java/org/zanata/maven/AbstractPullMojo.java index 4aeee35a..5602b2ed 100644 --- a/zanata-maven-plugin/src/main/java/org/zanata/maven/AbstractPullMojo.java +++ b/zanata-maven-plugin/src/main/java/org/zanata/maven/AbstractPullMojo.java @@ -27,6 +27,7 @@ import org.zanata.client.commands.pull.PullOptions; import org.zanata.client.commands.pull.RawPullCommand; +import com.google.common.base.Preconditions; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; /** @@ -107,11 +108,23 @@ public abstract class AbstractPullMojo extends */ private boolean continueAfterError = false; + /** + * Accepts integer from 0 to 100. Only pull translation documents which are + * at least PERCENT % (message based) completed. Please note specifying this + * option may cause longer time to pull for a large project. + * + * @parameter expression="${zanata.minDocPercent}" default-value="0" + */ + private int minDocPercent = 0; + /** * */ public AbstractPullMojo() { super(); + Preconditions + .checkArgument(minDocPercent >= 0 && minDocPercent <= 100, + "zanata.minDocPercent should be an integer from 0 to 100"); } public PushPullCommand initCommand() { @@ -174,4 +187,9 @@ public String getCommandName() { public boolean isAuthRequired() { return false; } + + @Override + public int getMinDocPercent() { + return minDocPercent; + } } diff --git a/zanata-rest-client/src/main/java/org/zanata/rest/client/StatisticsResourceClient.java b/zanata-rest-client/src/main/java/org/zanata/rest/client/StatisticsResourceClient.java index 7bc6a071..19db011d 100644 --- a/zanata-rest-client/src/main/java/org/zanata/rest/client/StatisticsResourceClient.java +++ b/zanata-rest-client/src/main/java/org/zanata/rest/client/StatisticsResourceClient.java @@ -22,6 +22,8 @@ package org.zanata.rest.client; import java.net.URI; +import java.util.ArrayList; +import java.util.List; import javax.ws.rs.DefaultValue; import org.zanata.rest.dto.stats.ContainerTranslationStatistics; @@ -59,10 +61,20 @@ public ContainerTranslationStatistics getStatistics(String projectSlug, .queryParam("detail", String.valueOf(includeDetails)) .queryParam("word", String.valueOf(includeWordStats)) .queryParams(asMultivaluedMap("locale", - Lists.newArrayList(locales))); + toLocaleList(locales))); return webResource.get(ContainerTranslationStatistics.class); } + private static List toLocaleList(String[] locales) { + List localesList; + if (locales == null) { + localesList = Lists.newArrayList(); + } else { + localesList = Lists.newArrayList(locales); + } + return localesList; + } + @Override public ContainerTranslationStatistics getStatistics(String projectSlug, String iterationSlug, String docId, @@ -77,7 +89,7 @@ public ContainerTranslationStatistics getStatistics(String projectSlug, .path(docId) .queryParam("word", String.valueOf(includeWordStats)) .queryParams(asMultivaluedMap("locale", - Lists.newArrayList(locales))); + toLocaleList(locales))); return webResource.get(ContainerTranslationStatistics.class); }