Bugfix: Handle the exception that throwed by blob when trigger test (#…

…550)
microsoft · Jul 26, 2023 · 29f42a6 · 29f42a6
1 parent 4528d2e
commit 29f42a6
Show file tree

Hide file tree

Showing 6 changed files with 48 additions and 20 deletions.
diff --git a/agent/src/main/java/com/microsoft/hydralab/agent/service/AgentWebSocketClientService.java b/agent/src/main/java/com/microsoft/hydralab/agent/service/AgentWebSocketClientService.java
@@ -21,12 +21,12 @@
 import com.microsoft.hydralab.common.monitor.MetricPushGateway;
 import com.microsoft.hydralab.common.util.Const;
 import com.microsoft.hydralab.common.util.GlobalConstant;
+import com.microsoft.hydralab.common.util.HydraLabRuntimeException;
 import com.microsoft.hydralab.common.util.ThreadUtils;
 import io.micrometer.core.instrument.MeterRegistry;
 import io.micrometer.core.instrument.Tags;
 import io.prometheus.client.exporter.BasicAuthHttpConnectionFactory;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -168,22 +168,28 @@ public void onMessage(Message message) {
     @NotNull
     private Message handleTestTaskRun(Message message) {
         Message response;
+        TestTask testTask = null;
+        TestTaskSpec testTaskSpec = null;
         try {
-            TestTaskSpec testTaskSpec = (TestTaskSpec) message.getBody();
+            testTaskSpec = (TestTaskSpec) message.getBody();
             testTaskSpec.updateWithDefaultValues();
             log.info("TestTaskSpec: {}", testTaskSpec);
-            TestTask testTask = testTaskEngineService.runTestTask(TestTask.convertToTestTask(testTaskSpec));
+            testTask = TestTask.convertToTestTask(testTaskSpec);
+            testTask = testTaskEngineService.runTestTask(testTask);
             if (testTask.getTestDevicesCount() <= 0) {
-                response = Message.error(message, 404, "No device meet the requirement on this agent: " + testTaskSpec);
-            } else {
-                response = Message.response(message, testTask);
-                response.setPath(Const.Path.TEST_TASK_UPDATE);
+                throw new HydraLabRuntimeException("No device meet the requirement on this agent: " + testTaskSpec);
             }
+            response = Message.response(message, testTask);
+            response.setPath(Const.Path.TEST_TASK_UPDATE);
         } catch (Exception e) {
             log.error(e.getMessage(), e);
-            response = Message.error(message, 500,
-                    String.format("%s\n%s\n%s", e.getClass().getName(), e.getMessage(),
-                            ExceptionUtils.getStackTrace(e)));
+            if (testTask == null) {
+                testTask.setId(testTaskSpec.testTaskId);
+            }
+            testTask.setStatus(TestTask.TestStatus.EXCEPTION);
+            testTask.setTestErrorMsg(e.getMessage());
+            response = Message.response(message, testTask);
+            response.setPath(Const.Path.TEST_TASK_RETRY);
         }
         return response;
     }
@@ -269,6 +275,8 @@ public void onOneDeviceComplete(TestTask testTask, TestRunDevice testRunDevice,
     @Override
     public void onDeviceOffline(TestTask testTask) {
         log.info("test task {} re-queue, send message", testTask.getId());
+        testTask.setStatus(TestTask.TestStatus.EXCEPTION);
+        testTask.setTestErrorMsg("Device offline");
         send(Message.ok(Const.Path.TEST_TASK_RETRY, testTask));
     }
 

diff --git a/agent/src/main/java/com/microsoft/hydralab/agent/service/TestTaskEngineService.java b/agent/src/main/java/com/microsoft/hydralab/agent/service/TestTaskEngineService.java
@@ -70,7 +70,7 @@ public TestTask runTestTask(TestTask testTask) {
 
         Set<TestRunDevice> chosenDevices = chooseDevices(testTask);
         if (chosenDevices.size() == 0) {
-            handleNoAvaiableDevice(testTask);
+            handleNoAvailableDevice(testTask);
             return testTask;
         }
 
@@ -93,14 +93,14 @@ public boolean doTask(TestRunDevice testRunDevice) throws Exception {
                 });
 
         if (deviceTaskControl == null) {
-            handleNoAvaiableDevice(testTask);
+            handleNoAvailableDevice(testTask);
         } else {
             testTask.setTestDevicesCount(deviceTaskControl.devices.size());
         }
         return testTask;
     }
 
-    private static void handleNoAvaiableDevice(TestTask testTask) {
+    private static void handleNoAvailableDevice(TestTask testTask) {
         testTask.setTestDevicesCount(0);
         testTask.setStatus(TestTask.TestStatus.CANCELED);
         log.warn("No available device found for {}, group devices: {}, task {} is canceled on this agent",

diff --git a/agent/src/main/java/com/microsoft/hydralab/agent/util/FileLoadUtil.java b/agent/src/main/java/com/microsoft/hydralab/agent/util/FileLoadUtil.java
@@ -9,14 +9,15 @@
 import com.microsoft.hydralab.common.file.StorageServiceClientProxy;
 import com.microsoft.hydralab.common.util.CommandOutputReceiver;
 import com.microsoft.hydralab.common.util.FileUtil;
+import com.microsoft.hydralab.common.util.FlowUtil;
+import com.microsoft.hydralab.common.util.HydraLabRuntimeException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
 import org.springframework.util.Assert;
 
 import javax.annotation.Resource;
 import java.io.File;
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
@@ -120,10 +121,10 @@ public void loadCommonFile(StorageFileInfo attachment) {
         }
     }
 
-    private File downloadFile(StorageFileInfo attachment, String location, String targetFilePath) throws IOException {
+    private File downloadFile(StorageFileInfo attachment, String location, String targetFilePath) throws Exception {
         File file = new File(location, targetFilePath);
         log.debug("download file from {} to {}", attachment.getBlobUrl(), file.getAbsolutePath());
-        storageServiceClientProxy.download(file, attachment);
+        FlowUtil.retryAndSleepWhenException(3, 10, () -> storageServiceClientProxy.download(file, attachment));
         return file;
     }
 
@@ -132,8 +133,9 @@ private File downloadFile(StorageFileInfo attachment) {
         try {
             file = downloadFile(attachment, appOptions.getTestPackageLocation(), attachment.getBlobPath());
             log.info("Download file success");
-        } catch (IOException e) {
+        } catch (Exception e) {
             log.error("Download file failed", e);
+            throw new HydraLabRuntimeException("Download file failed", e);
         }
         return file;
     }

diff --git a/center/src/main/java/com/microsoft/hydralab/center/service/DeviceAgentManagementService.java b/center/src/main/java/com/microsoft/hydralab/center/service/DeviceAgentManagementService.java
@@ -298,14 +298,13 @@ private void handleQualifiedAgentMessage(Message message, AgentSessionInfo saved
                 if (message.getBody() instanceof TestTask) {
                     TestTask testTask = (TestTask) message.getBody();
                     if (testTask.getRetryTime() == Const.AgentConfig.RETRY_TIME) {
-                        testTask.setStatus(TestTask.TestStatus.EXCEPTION);
-                        testTask.setTestErrorMsg("Device offline!");
                         testDataService.saveTestTaskData(testTask);
                     } else {
                         TestTaskSpec taskSpec = TestTask.convertToTestTaskSpec(testTask);
                         taskSpec.retryTime++;
                         testTaskService.addTask(taskSpec);
-                        cancelTestTaskById(testTask.getId(), "Retry time limit!");
+                        log.info("Retry task {} for {} time", testTask.getId(), taskSpec.retryTime);
+                        cancelTestTaskById(testTask.getId(), "Error happened:" + testTask.getTestErrorMsg() + ". Will cancel the task and retry.");
                         //run the task saved in queue
                         testTaskService.runTask();
                     }

diff --git a/common/src/main/java/com/microsoft/hydralab/common/entity/common/TestTask.java b/common/src/main/java/com/microsoft/hydralab/common/entity/common/TestTask.java
@@ -214,6 +214,7 @@ public static TestTaskSpec convertToTestTaskSpec(TestTask testTask) {
         testTaskSpec.disableRecording = testTask.isDisableRecording();
         testTaskSpec.enableNetworkMonitor = testTask.isEnableNetworkMonitor();
         testTaskSpec.networkMonitorRule = testTask.getNetworkMonitorRule();
+        testTaskSpec.retryTime = testTask.getRetryTime();
 
         return testTaskSpec;
     }

diff --git a/common/src/main/java/com/microsoft/hydralab/common/util/FlowUtil.java b/common/src/main/java/com/microsoft/hydralab/common/util/FlowUtil.java
@@ -42,4 +42,22 @@ public static boolean retryAndSleepWhenFalse(int count, int sleepSeconds, Callab
         }
         return false;
     }
+
+    public static boolean retryAndSleepWhenException(int count, int sleepSeconds, Callable predicate) throws Exception {
+        Exception toThrow = null;
+        while (count > 0) {
+            try {
+                predicate.call();
+                return true;
+            } catch (Exception e) {
+                toThrow = e;
+            }
+            ThreadUtils.safeSleep(sleepSeconds * 1000);
+            count--;
+        }
+        if (toThrow != null) {
+            throw toThrow;
+        }
+        return false;
+    }
 }