Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: pre-translate "file" param #759

Merged
merged 1 commit into from
Apr 2, 2024
Merged

feat: pre-translate "file" param #759

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/main/java/com/crowdin/cli/commands/Actions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,8 @@ NewAction<ProjectProperties, ClientTask> taskAdd(
NewAction<NoProperties, NoClient> checkNewVersion();

NewAction<PropertiesWithFiles, ProjectClient> preTranslate(
List<String> languageIds, Method method, Long engineId, String branchName, AutoApproveOption autoApproveOption, Boolean duplicateTranslations,
Boolean translateUntranslatedOnly, Boolean translateWithPerfectMatchOnly, boolean noProgress, boolean debug, boolean verbose, boolean plainView, List<String> labelNames);
List<String> languageIds, List<String> files, Method method, Long engineId, String branchName, AutoApproveOption autoApproveOption, Boolean duplicateTranslations,
Boolean translateUntranslatedOnly, Boolean translateWithPerfectMatchOnly, boolean noProgress, boolean plainView, List<String> labelNames);

NewAction<ProjectProperties, ProjectClient> branchAdd(String name, String title, String exportPattern, Priority priority);

Expand Down
8 changes: 4 additions & 4 deletions src/main/java/com/crowdin/cli/commands/actions/CliActions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,11 +222,11 @@ public NewAction<NoProperties, NoClient> checkNewVersion() {

@Override
public NewAction<PropertiesWithFiles, ProjectClient> preTranslate(
List<String> languageIds, Method method, Long engineId, String branchName, AutoApproveOption autoApproveOption, Boolean duplicateTranslations,
Boolean translateUntranslatedOnly, Boolean translateWithPerfectMatchOnly, boolean noProgress, boolean debug, boolean verbose, boolean plainView, List<String> labelNames
List<String> languageIds, List<String> files, Method method, Long engineId, String branchName, AutoApproveOption autoApproveOption, Boolean duplicateTranslations,
Boolean translateUntranslatedOnly, Boolean translateWithPerfectMatchOnly, boolean noProgress, boolean plainView, List<String> labelNames
) {
return new PreTranslateAction(languageIds, method, engineId, branchName, autoApproveOption, duplicateTranslations,
translateUntranslatedOnly, translateWithPerfectMatchOnly, noProgress, debug, verbose, plainView, labelNames);
return new PreTranslateAction(languageIds, files, method, engineId, branchName, autoApproveOption, duplicateTranslations,
translateUntranslatedOnly, translateWithPerfectMatchOnly, noProgress, plainView, labelNames);
}

@Override
Expand Down
209 changes: 103 additions & 106 deletions src/main/java/com/crowdin/cli/commands/actions/PreTranslateAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,17 @@
import com.crowdin.cli.client.ProjectClient;
import com.crowdin.cli.commands.NewAction;
import com.crowdin.cli.commands.Outputter;
import com.crowdin.cli.commands.functionality.*;
import com.crowdin.cli.commands.functionality.ProjectFilesUtils;
import com.crowdin.cli.commands.functionality.RequestBuilder;
import com.crowdin.cli.properties.PropertiesWithFiles;
import com.crowdin.cli.utils.PlaceholderUtil;
import com.crowdin.cli.utils.Utils;
import com.crowdin.cli.utils.console.ConsoleSpinner;
import com.crowdin.client.labels.model.Label;
import com.crowdin.client.languages.model.Language;
import com.crowdin.client.projectsgroups.model.Type;
import com.crowdin.client.sourcefiles.model.Branch;
import com.crowdin.client.sourcefiles.model.FileInfo;
import com.crowdin.client.translations.model.*;
import org.apache.commons.lang3.StringUtils;

import java.io.File;
import java.util.*;
import java.util.stream.Collectors;

Expand All @@ -28,25 +25,25 @@

class PreTranslateAction implements NewAction<PropertiesWithFiles, ProjectClient> {

private List<String> languageIds;
private Method method;
private Long engineId;
private String branchName;
private AutoApproveOption autoApproveOption;
private Boolean duplicateTranslations;
private Boolean translateUntranslatedOnly;
private Boolean translateWithPerfectMatchOnly;
private boolean noProgress;
private boolean debug;
private boolean verbose;
private boolean plainView;
private List<String> labelNames;
private final List<String> languageIds;
private final List<String> files;
private final Method method;
private final Long engineId;
private final String branchName;
private final AutoApproveOption autoApproveOption;
private final Boolean duplicateTranslations;
private final Boolean translateUntranslatedOnly;
private final Boolean translateWithPerfectMatchOnly;
private final boolean noProgress;
private final boolean plainView;
private final List<String> labelNames;

public PreTranslateAction(
List<String> languageIds, Method method, Long engineId, String branchName, AutoApproveOption autoApproveOption, Boolean duplicateTranslations,
Boolean translateUntranslatedOnly, Boolean translateWithPerfectMatchOnly, boolean noProgress, boolean debug, boolean verbose, boolean plainView, List<String> labelNames
List<String> languageIds, List<String> files, Method method, Long engineId, String branchName, AutoApproveOption autoApproveOption, Boolean duplicateTranslations,
Boolean translateUntranslatedOnly, Boolean translateWithPerfectMatchOnly, boolean noProgress, boolean plainView, List<String> labelNames
) {
this.languageIds = languageIds;
this.files = files;
this.method = method;
this.engineId = engineId;
this.branchName = branchName;
Expand All @@ -55,98 +52,98 @@ public PreTranslateAction(
this.translateUntranslatedOnly = translateUntranslatedOnly;
this.translateWithPerfectMatchOnly = translateWithPerfectMatchOnly;
this.noProgress = noProgress;
this.debug = debug;
this.verbose = verbose;
this.plainView = plainView;
this.labelNames = labelNames;
}

@Override
public void act(Outputter out, PropertiesWithFiles properties, ProjectClient client) {
CrowdinProjectFull project = ConsoleSpinner.execute(out, "message.spinner.fetching_project_info", "error.collect_project_info",
this.noProgress, this.plainView, () -> client.downloadFullProject(this.branchName));
this.noProgress, this.plainView, () -> client.downloadFullProject(this.branchName));
boolean isStringsBasedProject = Objects.equals(project.getType(), Type.STRINGS_BASED);

List<String> languages = this.prepareLanguageIds(project);
List<Long> labelIds = this.prepareLabelIds(out, client);

if (!isStringsBasedProject) {
List<Long> fileIds = this.prepareFileIds(out, properties, project);
if (fileIds == null || fileIds.isEmpty()) {
throw new RuntimeException(String.format(RESOURCE_BUNDLE.getString("error.no_files_found_for_pre_translate")));
if (isStringsBasedProject) {
if (files != null && !files.isEmpty()) {
throw new RuntimeException(RESOURCE_BUNDLE.getString("message.no_file_string_project"));
}
ApplyPreTranslationRequest request = RequestBuilder.applyPreTranslation(
languages, fileIds, method, engineId, autoApproveOption,
duplicateTranslations, translateUntranslatedOnly, translateWithPerfectMatchOnly, labelIds);
this.applyPreTranslation(out, client, request);
} else {
Branch branch = project.findBranchByName(branchName)
.orElseThrow(() -> new RuntimeException(RESOURCE_BUNDLE.getString("error.branch_required_string_project")));
.orElseThrow(() -> new RuntimeException(RESOURCE_BUNDLE.getString("error.branch_required_string_project")));
ApplyPreTranslationStringsBasedRequest request = RequestBuilder.applyPreTranslationStringsBased(
languages, Collections.singletonList(branch.getId()), method, engineId, autoApproveOption,
duplicateTranslations, translateUntranslatedOnly, translateWithPerfectMatchOnly, labelIds);
languages, Collections.singletonList(branch.getId()), method, engineId, autoApproveOption,
duplicateTranslations, translateUntranslatedOnly, translateWithPerfectMatchOnly, labelIds);
this.applyPreTranslationStringsBased(out, client, request);
return;
}

if (files == null || files.isEmpty()) {
throw new RuntimeException(RESOURCE_BUNDLE.getString("error.file_required"));
}

Optional<Branch> branch = Optional.ofNullable(branchName).flatMap(project::findBranchByName);

if (!branch.isPresent() && branchName != null) {
throw new RuntimeException(String.format(RESOURCE_BUNDLE.getString("message.branch_does_not_exist"), branchName));
}

List<FileInfo> fileInfos = project
.getFileInfos()
.stream().filter(f -> !branch.isPresent() || branch.get().getId().equals(f.getBranchId()))
.collect(Collectors.toList());
Map<String, FileInfo> paths = ProjectFilesUtils.buildFilePaths(project.getDirectories(), fileInfos);
boolean containsError = false;

List<Long> fileIds = new ArrayList<>();

for (String file : files) {
if (!paths.containsKey(file)) {
if (files.size() > 1) {
containsError = true;
out.println(WARNING.withIcon(String.format(RESOURCE_BUNDLE.getString("error.file_not_exists"), file)));
continue;
} else {
throw new RuntimeException(String.format(RESOURCE_BUNDLE.getString("error.file_not_exists"), file));
}
}
Long fileId = paths.get(file).getId();
fileIds.add(fileId);
}

if (fileIds.isEmpty()) {
throw new RuntimeException(String.format(RESOURCE_BUNDLE.getString("error.no_files_found_for_pre_translate")));
}

ApplyPreTranslationRequest request = RequestBuilder.applyPreTranslation(
languages, fileIds, method, engineId, autoApproveOption,
duplicateTranslations, translateUntranslatedOnly, translateWithPerfectMatchOnly, labelIds);
this.applyPreTranslation(out, client, request);

if (containsError) {
throw new RuntimeException();
}
}

private List<String> prepareLanguageIds(CrowdinProjectInfo projectInfo) {
List<String> projectLanguages = projectInfo.getProjectLanguages(false).stream()
.map(Language::getId)
.collect(Collectors.toList());
.map(Language::getId)
.collect(Collectors.toList());
if (languageIds.size() == 1 && BaseCli.ALL.equalsIgnoreCase(languageIds.get(0))) {
return projectLanguages;
} else {
String wrongLanguageIds = languageIds.stream()
.filter(langId -> !projectLanguages.contains(langId))
.map(id -> "'" + id + "'")
.collect(Collectors.joining(", "));
.filter(langId -> !projectLanguages.contains(langId))
.map(id -> "'" + id + "'")
.collect(Collectors.joining(", "));
if (!wrongLanguageIds.isEmpty()) {
throw new RuntimeException(
String.format(RESOURCE_BUNDLE.getString("error.languages_not_exist"), wrongLanguageIds));
String.format(RESOURCE_BUNDLE.getString("error.languages_not_exist"), wrongLanguageIds));
}
return languageIds;
}
}

private List<Long> prepareFileIds(Outputter out, PropertiesWithFiles pb, CrowdinProjectFull project) {
Map<String, FileInfo> paths = ProjectFilesUtils.buildFilePaths(project.getDirectories(), project.getBranches(), project.getFileInfos());
PlaceholderUtil placeholderUtil = new PlaceholderUtil(project.getSupportedLanguages(), project.getProjectLanguages(false), pb.getBasePath());
List<String> sourcePaths = pb.getFiles().stream()
.flatMap(file -> {
List<String> sources = SourcesUtils.getFiles(pb.getBasePath(), file.getSource(), file.getIgnore(), placeholderUtil)
.map(File::getAbsolutePath)
.collect(Collectors.toList());
String commonPath = (pb.getPreserveHierarchy()) ? "" : SourcesUtils.getCommonPath(sources, pb.getBasePath());
return sources.stream()
.map(source -> (file.getDest() != null)
? PropertiesBeanUtils.prepareDest(file.getDest(), StringUtils.removeStart(source, pb.getBasePath()), placeholderUtil) : StringUtils.removeStart(source, pb.getBasePath() + commonPath))
.map(source -> (branchName != null ? branchName + Utils.PATH_SEPARATOR : "") + source);
})
.distinct()
.collect(Collectors.toList());
List<String> onlyLocalSources = new ArrayList<>();
List<String> foundSources = new ArrayList<>();
for (String sourcePath : sourcePaths) {
if (paths.containsKey(sourcePath)) {
foundSources.add(sourcePath);
} else {
onlyLocalSources.add(sourcePath);
}
}
if (!onlyLocalSources.isEmpty()) {
if (verbose) {
out.println(WARNING.withIcon(String.format(RESOURCE_BUNDLE.getString("message.pre_translate.local_files_message_verbose"), sourcePaths.size(), onlyLocalSources.size())));
onlyLocalSources.forEach(source -> out.println(String.format(RESOURCE_BUNDLE.getString("message.item_list"), source)));
} else {
out.println(WARNING.withIcon(String.format(RESOURCE_BUNDLE.getString("message.pre_translate.local_files_message"), sourcePaths.size(), onlyLocalSources.size())));
}
}
return foundSources.stream()
.map(paths::get)
.map(FileInfo::getId)
.collect(Collectors.toList());
}

private List<Long> prepareLabelIds(Outputter out, ProjectClient client) {
if (labelNames != null && !labelNames.isEmpty()) {
Map<String, Long> labels = client.listLabels().stream()
Expand All @@ -170,37 +167,37 @@ private List<Long> prepareLabelIds(Outputter out, ProjectClient client) {

private void applyPreTranslation(Outputter out, ProjectClient client, ApplyPreTranslationRequest request) {
ConsoleSpinner.execute(
out,
"message.spinner.pre_translate",
"error.spinner.pre_translate",
this.noProgress,
this.plainView,
() -> {
PreTranslationStatus preTranslationStatus = client.startPreTranslation(request);
preTranslationStatus = handlePreTranslationStatus(client, preTranslationStatus);

ConsoleSpinner.update(String.format(RESOURCE_BUNDLE.getString("message.spinner.pre_translate_done"), 100));

return preTranslationStatus;
}
out,
"message.spinner.pre_translate",
"error.spinner.pre_translate",
this.noProgress,
this.plainView,
() -> {
PreTranslationStatus preTranslationStatus = client.startPreTranslation(request);
preTranslationStatus = handlePreTranslationStatus(client, preTranslationStatus);

ConsoleSpinner.update(String.format(RESOURCE_BUNDLE.getString("message.spinner.pre_translate_done"), 100));

return preTranslationStatus;
}
);
}

private void applyPreTranslationStringsBased(Outputter out, ProjectClient client, ApplyPreTranslationStringsBasedRequest request) {
ConsoleSpinner.execute(
out,
"message.spinner.pre_translate",
"error.spinner.pre_translate",
this.noProgress,
this.plainView,
() -> {
PreTranslationStatus preTranslationStatus = client.startPreTranslationStringsBased(request);
preTranslationStatus = handlePreTranslationStatus(client, preTranslationStatus);

ConsoleSpinner.update(String.format(RESOURCE_BUNDLE.getString("message.spinner.pre_translate_done"), 100));

return preTranslationStatus;
}
out,
"message.spinner.pre_translate",
"error.spinner.pre_translate",
this.noProgress,
this.plainView,
() -> {
PreTranslationStatus preTranslationStatus = client.startPreTranslationStringsBased(request);
preTranslationStatus = handlePreTranslationStatus(client, preTranslationStatus);

ConsoleSpinner.update(String.format(RESOURCE_BUNDLE.getString("message.spinner.pre_translate_done"), 100));

return preTranslationStatus;
}
);
}

Expand Down
14 changes: 12 additions & 2 deletions src/main/java/com/crowdin/cli/commands/picocli/PreTranslateSubcommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@
import com.crowdin.cli.commands.Actions;
import com.crowdin.cli.commands.NewAction;
import com.crowdin.cli.properties.PropertiesWithFiles;
import com.crowdin.cli.utils.Utils;
import com.crowdin.client.translations.model.AutoApproveOption;
import com.crowdin.client.translations.model.Method;
import org.apache.commons.lang3.StringUtils;
import picocli.CommandLine;

import java.util.ArrayList;
Expand All @@ -25,6 +27,9 @@ public class PreTranslateSubcommand extends ActCommandWithFiles {
@CommandLine.Option(names = {"-l", "--language"}, paramLabel = "...", defaultValue = BaseCli.ALL, order = -2)
protected List<String> languageIds;

@CommandLine.Option(names = {"--file"}, paramLabel = "...", order = -2)
protected List<String> files;

@CommandLine.Option(names = {"--method"}, paramLabel = "...", required = true, order = -2)
protected Method method;

Expand Down Expand Up @@ -68,6 +73,7 @@ protected final boolean isAnsi() {
protected NewAction<PropertiesWithFiles, ProjectClient> getAction(Actions actions) {
return actions.preTranslate(
languageIds,
files,
method,
engineId,
branch,
Expand All @@ -76,8 +82,6 @@ protected NewAction<PropertiesWithFiles, ProjectClient> getAction(Actions action
translateUntranslatedOnly,
translateWithPerfectMatchOnly,
noProgress,
debug,
isVerbose,
plainView,
labelNames
);
Expand Down Expand Up @@ -105,6 +109,12 @@ protected List<String> checkOptions() {
if (autoApproveOption != null && !autoApproveOptionWrapper.containsKey(autoApproveOption)) {
errors.add(RESOURCE_BUNDLE.getString("error.pre_translate.auto_approve_option"));
}
if (files != null) {
for (int i = 0; i < files.size(); i++) {
String normalizedFile = StringUtils.removeStart(Utils.normalizePath(files.get(i)), Utils.PATH_SEPARATOR);
files.set(i, normalizedFile);
}
}
return errors;
}
}
1 change: 1 addition & 0 deletions src/main/resources/messages/messages.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,7 @@ crowdin.comment.resolve.id=Numeric ID of the string comment
crowdin.pre-translate.usage.description=Pre-translate files via Machine Translation (MT) or Translation Memory (TM)
crowdin.pre-translate.usage.customSynopsis=@|fg(green) crowdin pre-translate|@ [CONFIG OPTIONS] [OPTIONS]
crowdin.pre-translate.language=Languages to which pre-translation should be applied. Default: ALL
crowdin.pre-translate.file=Specify a file to pre-translate (multiple files could be specified)
crowdin.pre-translate.method=Defines pre-translation method. Supported values: mt, tm
crowdin.pre-translate.engine-id=Machine Translation engine Identifier
crowdin.pre-translate.auto-approve-option=Defines which translations added by TM pre-translation should be auto-approved. Supported values: all, except-auto-substituted, perfect-match-only. Default: none
Expand Down
7 changes: 6 additions & 1 deletion website/docs/releases/migration-4.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,4 +123,9 @@ After:
### Download targets

* removed `crowdin download targets` command
* removed `targets` from configuration file
* removed `targets` from configuration file

### Pre-translate

* added `file` parameter (required for file-based projects)
* now `file` value(s) determines which files to pre-translate