Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: retries for uploads #806

Merged
merged 2 commits into from
May 22, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions src/main/java/com/crowdin/cli/client/CrowdinClientCore.java
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,6 @@ protected static <T> List<T> executeRequestFullList(BiFunction<Integer, Integer,
return directories;
}

protected static <T> T executeRequestWithPossibleRetry(Map<BiPredicate<String, String>, ResponseException> errorHandlers, Supplier<T> request) throws ResponseException {
return executeRequestWithPossibleRetries(errorHandlers, request, 2, defaultMillisToRetry);
}

protected static <T> T executeRequestWithPossibleRetries(Map<BiPredicate<String, String>, ResponseException> errorHandlers, Supplier<T> request, int maxAttempts, long millisToRetry) throws ResponseException {
if (maxAttempts < 1) {
throw new MaxNumberOfRetriesException();
Expand All @@ -93,6 +89,7 @@ protected static <T> T executeRequestWithPossibleRetries(Map<BiPredicate<String,
} catch (InterruptedException ie) {
// ignore
}
System.out.printf("Attempting to retry the request due to the error: %s%n", e.getMessage());
return executeRequestWithPossibleRetries(errorHandlers, request, maxAttempts - 1, millisToRetry);
}
}
Expand Down
59 changes: 42 additions & 17 deletions src/main/java/com/crowdin/cli/client/CrowdinProjectClient.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.crowdin.cli.client;

import com.crowdin.cli.utils.Utils;
import com.crowdin.client.branches.model.*;
import com.crowdin.client.core.model.PatchRequest;
import com.crowdin.client.labels.model.AddLabelRequest;
Expand Down Expand Up @@ -212,10 +213,16 @@ public Long uploadStorage(String fileName, InputStream content) throws Response
Map<BiPredicate<String, String>, ResponseException> errorHandlers = new LinkedHashMap<BiPredicate<String, String>, ResponseException>() {{
put((code, message) -> StringUtils.containsAny(message, "streamIsEmpty", "Stream size is null. Not empty content expected"),
new EmptyFileException("Not empty content expected"));
put((code, message) -> Utils.isServerErrorCode(code), new RepeatException("Server Error"));
put((code, message) -> message.contains("Request aborted"), new RepeatException("Request aborted"));
put((code, message) -> message.contains("Connection reset"), new RepeatException("Connection reset"));
}};
Storage storage = executeRequest(errorHandlers, () -> this.client.getStorageApi()
.addStorage(fileName, content)
.getData());
Storage storage = executeRequestWithPossibleRetries(
errorHandlers,
() -> this.client.getStorageApi().addStorage(fileName, content).getData(),
3,
2 * 1000
);
return storage.getId();
}

Expand Down Expand Up @@ -244,28 +251,38 @@ public void deleteDirectory(Long directoryId) {
@Override
public void updateSource(Long sourceId, UpdateFileRequest request) throws ResponseException {
Map<BiPredicate<String, String>, ResponseException> errorHandlers = new LinkedHashMap<BiPredicate<String, String>, ResponseException>() {{
put((code, message) -> message.contains("File from storage with id #" + request.getStorageId() + " was not found"), new RepeatException());
put((code, message) -> message.contains("File from storage with id #" + request.getStorageId() + " was not found"), new RepeatException("File not found in the storage"));
put((code, message) -> Utils.isServerErrorCode(code), new RepeatException("Server Error"));
put((code, message) -> message.contains("Request aborted"), new RepeatException("Request aborted"));
put((code, message) -> message.contains("Connection reset"), new RepeatException("Connection reset"));
put((code, message) -> StringUtils.contains(message, "Invalid SRX specified"), new ResponseException("Invalid SRX file specified"));
put((code, message) -> code.equals("409"), new FileInUpdateException());
}};
executeRequestWithPossibleRetry(
executeRequestWithPossibleRetries(
errorHandlers,
() -> this.client.getSourceFilesApi()
.updateOrRestoreFile(this.projectId, sourceId, request));
() -> this.client.getSourceFilesApi().updateOrRestoreFile(this.projectId, sourceId, request),
3,
2 * 1000
);
}

@Override
public FileInfo addSource(AddFileRequest request) throws ResponseException {
Map<BiPredicate<String, String>, ResponseException> errorHandlers = new LinkedHashMap<BiPredicate<String, String>, ResponseException>() {{
put((code, message) -> message.contains("File from storage with id #" + request.getStorageId() + " was not found"), new RepeatException());
put((code, message) -> message.contains("File from storage with id #" + request.getStorageId() + " was not found"), new RepeatException("File not found in the storage"));
put((code, message) -> Utils.isServerErrorCode(code), new RepeatException("Server Error"));
put((code, message) -> message.contains("Request aborted"), new RepeatException("Request aborted"));
put((code, message) -> message.contains("Connection reset"), new RepeatException("Connection reset"));
put((code, message) -> StringUtils.contains(message, "Name must be unique"), new ExistsResponseException());
put((code, message) -> StringUtils.contains(message, "Invalid SRX specified"), new ResponseException("Invalid SRX file specified"));
put((code, message) -> StringUtils.containsAny(message, "isEmpty", "Value is required and can't be empty"), new EmptyFileException("Value is required and can't be empty"));
}};
return executeRequestWithPossibleRetry(
return executeRequestWithPossibleRetries(
errorHandlers,
() -> this.client.getSourceFilesApi()
.addFile(this.projectId, request).getData());
() -> this.client.getSourceFilesApi().addFile(this.projectId, request).getData(),
3,
2 * 1000
);
}

@Override
Expand Down Expand Up @@ -303,12 +320,17 @@ public void uploadTranslations(String languageId, UploadTranslationsRequest requ
put((code, message) -> code.equals("0") && message.equals("File is not allowed for language"),
new WrongLanguageException());
put((code, message) -> message.contains("File from storage with id #" + request.getStorageId() + " was not found"),
new RepeatException());
new RepeatException("File not found in the storage"));
put((code, message) -> Utils.isServerErrorCode(code), new RepeatException("Server Error"));
put((code, message) -> message.contains("Request aborted"), new RepeatException("Request aborted"));
put((code, message) -> message.contains("Connection reset"), new RepeatException("Connection reset"));
}};
executeRequestWithPossibleRetry(
executeRequestWithPossibleRetries(
errorhandlers,
() -> this.client.getTranslationsApi()
.uploadTranslations(this.projectId, languageId, request));
() -> this.client.getTranslationsApi().uploadTranslations(this.projectId, languageId, request),
3,
2 * 1000
);
}

@Override
Expand All @@ -321,13 +343,16 @@ public void uploadTranslationStringsBased(String languageId, UploadTranslationsS
public ProjectBuild startBuildingTranslation(BuildProjectTranslationRequest request) throws ResponseException {
Map<BiPredicate<String, String>, ResponseException> errorHandler = new LinkedHashMap<BiPredicate<String, String>, ResponseException>() {{
put((code, message) -> code.equals("409") && message.contains("Another build is currently in progress"),
new RepeatException());
new RepeatException("Another build is currently in progress"));
put((code, message) -> Utils.isServerErrorCode(code), new RepeatException("Server Error"));
put((code, message) -> message.contains("Request aborted"), new RepeatException("Request aborted"));
put((code, message) -> message.contains("Connection reset"), new RepeatException("Connection reset"));
}};
return executeRequestWithPossibleRetries(
errorHandler,
() -> this.client.getTranslationsApi().buildProjectTranslation(this.projectId, request).getData(),
3,
60 * 100
6 * 1000
);
}

Expand Down
4 changes: 4 additions & 0 deletions src/main/java/com/crowdin/cli/client/RepeatException.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,8 @@ class RepeatException extends ResponseException {
public RepeatException() {
super();
}

public RepeatException(String message) {
super(message);
}
}
10 changes: 10 additions & 0 deletions src/main/java/com/crowdin/cli/utils/Utils.java
Original file line number Diff line number Diff line change
Expand Up @@ -182,4 +182,14 @@ public static String encodeURL(@NonNull String toEncode) {
public static String toSingleLineString(String str) {
return str.replaceAll("[\r\n]+", " ");
}

public static boolean isServerErrorCode(String code) {
int statusCode;
try {
statusCode = Integer.parseInt(code);
} catch (NumberFormatException e) {
return false;
}
return statusCode >= 500 && statusCode < 600;
}
}
Loading