Skip to content

Commit

Permalink
Extract retry methods of API calling (#590)
Browse files Browse the repository at this point in the history
  • Loading branch information
@olivershen-wow
olivershen-wow authored Sep 12, 2023
1 parent 04ec4ea commit 7e7a1ce
Show file tree
Hide file tree
Showing 3 changed files with 155 additions and 107 deletions.
8 changes: 8 additions & 0 deletions gradle_plugin/src/main/groovy/com/microsoft/hydralab/utils/CommonUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,4 +107,12 @@ public static void assertTrue(boolean beTrue, String msg, Object data) {
throw new IllegalStateException(msg + (data == null ? "" : ": " + data));
}
}

public static void safeSleep(long millis) {
try {
Thread.sleep(millis);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
36 changes: 36 additions & 0 deletions gradle_plugin/src/main/groovy/com/microsoft/hydralab/utils/FlowUtil.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

package com.microsoft.hydralab.utils;

import okhttp3.Response;

import java.util.concurrent.Callable;

/**
* @author Li Shen
* @date 9/11/2023
*/

public class FlowUtil {
public static Response httpRetryAndSleepWhenException(int count, int sleepSeconds, Callable<Response> predicate) throws Exception {
Response res = null;
Exception toThrow = null;
while (count > 0) {
try {
res = predicate.call();
return res;
} catch (Exception e) {
toThrow = e;
}
CommonUtils.safeSleep(sleepSeconds * 1000L);
count--;
}
if (toThrow != null) {
throw toThrow;
}
return res;
}


}
218 changes: 111 additions & 107 deletions gradle_plugin/src/main/groovy/com/microsoft/hydralab/utils/HydraLabAPIClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;

import static com.microsoft.hydralab.utils.CommonUtils.GSON;
import static com.microsoft.hydralab.utils.CommonUtils.assertNotNull;
Expand All @@ -50,22 +51,22 @@ public void checkCenterAlive(HydraLabAPIConfig apiConfig) {
.url(apiConfig.checkCenterAliveUrl())
.build();
OkHttpClient clientToUse = client;
Response response = null;
try {
response = clientToUse.newCall(req).execute();
int waitingRetry = httpFailureRetryTimes;
while (!response.isSuccessful() && waitingRetry > 0) {
printlnf("##[warning]Check center alive failed, remaining retry times: %d\nHttp code: %d\nHttp message: %s", waitingRetry, response.code(), response.message());
response = clientToUse.newCall(req).execute();
waitingRetry--;
}

assertTrue(response.isSuccessful(), "check center alive", response);
AtomicInteger waitingRetry = new AtomicInteger(httpFailureRetryTimes);
try (Response response = FlowUtil.httpRetryAndSleepWhenException(httpFailureRetryTimes, 1,
() -> {
Response res = clientToUse.newCall(req).execute();
if (!res.isSuccessful()) {
waitingRetry.getAndDecrement();
printlnf("##[warning]Check center alive failed, remaining retry times: %d\nHttp code: %d\nHttp message: %s", waitingRetry, res.code(),
res.message());
throw new IllegalStateException("Check center alive: " + res);
}
return res;
}
)) {
printlnf("Center is alive, continue on requesting API...");
} catch (Exception e) {
throw new RuntimeException("check center alive fail: " + e.getMessage(), e);
} finally {
response.close();
}
}

Expand Down Expand Up @@ -95,17 +96,19 @@ public String uploadApp(HydraLabAPIConfig apiConfig, TestConfig testConfig, Stri
.post(multipartBodyBuilder.build())
.build();
OkHttpClient clientToUse = client;
Response response = null;
try {
response = clientToUse.newCall(req).execute();
int waitingRetry = httpFailureRetryTimes;
while (!response.isSuccessful() && waitingRetry > 0) {
printlnf("##[warning]Upload App failed, remaining retry times: %d\nHttp code: %d\nHttp message: %s", waitingRetry, response.code(), response.message());
response = clientToUse.newCall(req).execute();
waitingRetry--;
}

assertTrue(response.isSuccessful(), "upload App", response);
AtomicInteger waitingRetry = new AtomicInteger(httpFailureRetryTimes);
try (Response response = FlowUtil.httpRetryAndSleepWhenException(httpFailureRetryTimes, 1,
() -> {
Response res = clientToUse.newCall(req).execute();
if (!res.isSuccessful()) {
waitingRetry.getAndDecrement();
printlnf("##[warning]Upload App failed, remaining retry times: %d\nHttp code: %d\nHttp message: %s", waitingRetry, res.code(),
res.message());
throw new IllegalStateException("Upload app: " + res);
}
return res;
}
)) {
ResponseBody body = response.body();

assertNotNull(body, response + ": upload App ResponseBody");
Expand All @@ -117,8 +120,6 @@ public String uploadApp(HydraLabAPIConfig apiConfig, TestConfig testConfig, Stri
return jsonObject.getAsJsonObject("content").get("id").getAsString();
} catch (Exception e) {
throw new RuntimeException("upload App fail: " + e.getMessage(), e);
} finally {
response.close();
}
}

Expand Down Expand Up @@ -147,25 +148,25 @@ public JsonObject addAttachment(HydraLabAPIConfig apiConfig, String testFileSetI
.post(multipartBodyBuilder.build())
.build();
OkHttpClient clientToUse = client;
Response response = null;
try {
response = clientToUse.newCall(req).execute();
int waitingRetry = httpFailureRetryTimes;
while (!response.isSuccessful() && waitingRetry > 0) {
printlnf("##[warning]Add attachments failed, remaining retry times: %d\nHttp code: %d\nHttp message: %s", waitingRetry, response.code(), response.message());
response = clientToUse.newCall(req).execute();
waitingRetry--;
}

assertTrue(response.isSuccessful(), "Add attachments", response);
AtomicInteger waitingRetry = new AtomicInteger(httpFailureRetryTimes);
try (Response response = FlowUtil.httpRetryAndSleepWhenException(httpFailureRetryTimes, 1,
() -> {
Response res = clientToUse.newCall(req).execute();
if (!res.isSuccessful()) {
waitingRetry.getAndDecrement();
printlnf("##[warning]Add attachments failed, remaining retry times: %d\nHttp code: %d\nHttp message: %s", waitingRetry, res.code(),
res.message());
throw new IllegalStateException("Add attachments: " + res);
}
return res;
}
)) {
ResponseBody body = response.body();
assertNotNull(body, response + ": Add attachments ResponseBody");

return GSON.fromJson(body.string(), JsonObject.class);
} catch (Exception e) {
throw new RuntimeException("Add attachments fail: " + e.getMessage(), e);
} finally {
response.close();
}
}

Expand All @@ -179,17 +180,20 @@ public String generateAccessKey(HydraLabAPIConfig apiConfig, TestConfig testConf
.build();
OkHttpClient clientToUse = client;
JsonObject jsonObject = null;
Response response = null;
try {
response = clientToUse.newCall(req).execute();
int waitingRetry = httpFailureRetryTimes;
while (!response.isSuccessful() && waitingRetry > 0) {
printlnf("##[warning]Generate accessKey failed, remaining retry times: %d\nHttp code: %d\nHttp message: %s", waitingRetry, response.code(), response.message());
response = clientToUse.newCall(req).execute();
waitingRetry--;
}
AtomicInteger waitingRetry = new AtomicInteger(httpFailureRetryTimes);
try (Response response = FlowUtil.httpRetryAndSleepWhenException(httpFailureRetryTimes, 1,
() -> {
Response res = clientToUse.newCall(req).execute();
if (!res.isSuccessful()) {
waitingRetry.getAndDecrement();
printlnf("##[warning]Generate accessKey failed, remaining retry times: %d\nHttp code: %d\nHttp message: %s", waitingRetry, res.code(),
res.message());
throw new IllegalStateException("Generate accessKey: " + res);
}
return res;
}
)) {

assertTrue(response.isSuccessful(), "generate accessKey", response);
ResponseBody body = response.body();

assertNotNull(body, response + ": generateAccessKey ResponseBody");
Expand All @@ -204,8 +208,6 @@ public String generateAccessKey(HydraLabAPIConfig apiConfig, TestConfig testConf
printlnf("##[warning]Request generateAccess failed: " + jsonObject.toString());
return "";
// throw new RuntimeException("generate accessKey fail: " + e.getMessage(), e);
} finally {
response.close();
}
}

Expand Down Expand Up @@ -273,28 +275,27 @@ public JsonObject triggerTestRun(TestConfig testConfig, HydraLabAPIConfig apiCon
.url(apiConfig.getRunTestUrl())
.post(jsonBody).build();
OkHttpClient clientToUse = client;
Response response = null;
try {
response = clientToUse.newCall(req).execute();
int waitingRetry = httpFailureRetryTimes;
while (!response.isSuccessful() && waitingRetry > 0) {
printlnf("##[warning]Trigger test running failed, remaining retry times: %d\nHttp code: %d\nHttp message: %s", waitingRetry, response.code(), response.message());
response = clientToUse.newCall(req).execute();
waitingRetry--;
}

assertTrue(response.isSuccessful(), "trigger test running", response);
AtomicInteger waitingRetry = new AtomicInteger(httpFailureRetryTimes);
try (Response response = FlowUtil.httpRetryAndSleepWhenException(httpFailureRetryTimes, 1,
() -> {
Response res = clientToUse.newCall(req).execute();
if (!res.isSuccessful()) {
waitingRetry.getAndDecrement();
printlnf("##[warning]Trigger test running failed, remaining retry times: %d\nHttp code: %d\nHttp message: %s", waitingRetry, res.code(),
res.message());
throw new IllegalStateException("Trigger test running: " + res);
}
return res;
}
)) {
ResponseBody body = response.body();
assertNotNull(body, response + ": triggerTestRun ResponseBody");
String string = body.string();
printlnf("RunningTestJson: %s", maskCred(string));
JsonObject jsonObject = GSON.fromJson(string, JsonObject.class);

return jsonObject;
return GSON.fromJson(string, JsonObject.class);
} catch (Exception e) {
throw new RuntimeException("trigger test running fail: " + e.getMessage(), e);
} finally {
response.close();
}
}

Expand All @@ -306,17 +307,19 @@ public TestTask getTestStatus(HydraLabAPIConfig apiConfig, String testTaskId) {
.url(apiConfig.getTestStatusUrl(testTaskId))
.build();
OkHttpClient clientToUse = client;
Response response = null;
try {
response = clientToUse.newCall(req).execute();
int waitingRetry = httpFailureRetryTimes;
while (!response.isSuccessful() && waitingRetry > 0) {
printlnf("##[warning]Get test status failed, remaining retry times: %d\nHttp code: %d\nHttp message: %s", waitingRetry, response.code(), response.message());
response = clientToUse.newCall(req).execute();
waitingRetry--;
}

assertTrue(response.isSuccessful(), "get test status", response);
AtomicInteger waitingRetry = new AtomicInteger(httpFailureRetryTimes);
try (Response response = FlowUtil.httpRetryAndSleepWhenException(httpFailureRetryTimes, 1,
() -> {
Response res = clientToUse.newCall(req).execute();
if (!res.isSuccessful()) {
waitingRetry.getAndDecrement();
printlnf("##[warning]Get test status failed, remaining retry times: %d\nHttp code: %d\nHttp message: %s", waitingRetry, res.code(),
res.message());
throw new IllegalStateException("Get test status: " + res);
}
return res;
}
)) {
ResponseBody body = response.body();
assertNotNull(body, response + ": getTestStatus ResponseBody");
JsonObject jsonObject = GSON.fromJson(body.string(), JsonObject.class);
Expand All @@ -331,8 +334,6 @@ public TestTask getTestStatus(HydraLabAPIConfig apiConfig, String testTaskId) {
return result;
} catch (Exception e) {
throw new RuntimeException("get test status fail: " + e.getMessage(), e);
} finally {
response.close();
}
}

Expand All @@ -342,17 +343,19 @@ public String getBlobSAS(HydraLabAPIConfig apiConfig) {
.url(apiConfig.getBlobSASUrl())
.build();
OkHttpClient clientToUse = client;
Response response = null;
try {
response = clientToUse.newCall(req).execute();
int waitingRetry = httpFailureRetryTimes;
while (!response.isSuccessful() && waitingRetry > 0) {
printlnf("##[warning]Get Blob SAS failed, remaining retry times: %d\nHttp code: %d\nHttp message: %s", waitingRetry, response.code(), response.message());
response = clientToUse.newCall(req).execute();
waitingRetry--;
}

assertTrue(response.isSuccessful(), "Get Blob SAS", response);
AtomicInteger waitingRetry = new AtomicInteger(httpFailureRetryTimes);
try (Response response = FlowUtil.httpRetryAndSleepWhenException(httpFailureRetryTimes, 1,
() -> {
Response res = clientToUse.newCall(req).execute();
if (!res.isSuccessful()) {
waitingRetry.getAndDecrement();
printlnf("##[warning]Get Blob SAS failed, remaining retry times: %d\nHttp code: %d\nHttp message: %s", waitingRetry, res.code(),
res.message());
throw new IllegalStateException("Get Blob SAS: " + res);
}
return res;
}
)) {
ResponseBody body = response.body();

assertNotNull(body, response + ": Blob SAS");
Expand All @@ -364,14 +367,14 @@ public String getBlobSAS(HydraLabAPIConfig apiConfig) {
return jsonObject.getAsJsonObject("content").get("signature").getAsString();
} catch (Exception e) {
throw new RuntimeException("Get Blob SAS fail: " + e.getMessage(), e);
} finally {
response.close();
}
}

public void downloadToFile(String fileUrl, File file) {
Request req = new Request.Builder().get().url(fileUrl).build();
try (Response response = client.newCall(req).execute()) {
try (Response response = FlowUtil.httpRetryAndSleepWhenException(httpFailureRetryTimes, 1,
() -> client.newCall(req).execute()
)) {
if (!response.isSuccessful()) {
return;
}
Expand All @@ -381,7 +384,7 @@ public void downloadToFile(String fileUrl, File file) {
try (FileOutputStream fos = new FileOutputStream(file)) {
IOUtils.copy(response.body().byteStream(), fos);
}
} catch (IOException e) {
} catch (Exception e) {
e.printStackTrace();
}
}
Expand All @@ -394,21 +397,22 @@ public void cancelTestTask(HydraLabAPIConfig apiConfig, String testTaskId, Strin
.url(String.format(apiConfig.getCancelTestTaskUrl(), testTaskId, reason))
.build();
OkHttpClient clientToUse = client;
Response response = null;
try {
response = clientToUse.newCall(req).execute();
int waitingRetry = httpFailureRetryTimes;
while (!response.isSuccessful() && waitingRetry > 0) {
printlnf("##[warning]Cancel test task failed, remaining retry times: %d\nHttp code: %d\nHttp message: %s", waitingRetry, response.code(), response.message());
response = clientToUse.newCall(req).execute();
waitingRetry--;
}

assertTrue(response.isSuccessful(), "cancel test task", response);
AtomicInteger waitingRetry = new AtomicInteger(httpFailureRetryTimes);
try (Response response = FlowUtil.httpRetryAndSleepWhenException(httpFailureRetryTimes, 1,
() -> {
Response res = clientToUse.newCall(req).execute();
if (!res.isSuccessful()) {
waitingRetry.getAndDecrement();
printlnf("##[warning]Cancel test task failed, remaining retry times: %d\nHttp code: %d\nHttp message: %s", waitingRetry, res.code(),
res.message());
throw new IllegalStateException("Cancel test task: " + res);
}
return res;
}
)) {
printlnf("Test task canceled");
} catch (Exception e) {
throw new RuntimeException("cancel test task fail: " + e.getMessage(), e);
} finally {
response.close();
}
}
}

0 comments on commit 7e7a1ce

Please sign in to comment.