Add performance test classes to provide interfaces and name definitio…

common/doc/UML/perf_design.puml

@@ -0,0 +1,164 @@
+@startuml
+'https://plantuml.com/class-diagram'
+
+package "sdk" {
+
+    class PerformanceTestSpec {
+        -String inspector
+        -String appId
+        -String deviceId
+        -String name
+    }
+    interface PerformanceInspector {
+        void initialize(PerformanceTestSpec performanceTestSpec)
+        PerformanceInspectionResult inspect(PerformanceTestSpec performanceTestSpec)
+        PerformanceTestResult parse(List<PerformanceInspectionResult>)
+    }
+
+    interface IPerformanceInspectionService {
+        void reset(PerformanceInspection);
+        PerformanceInspectionResult inspect(PerformanceInspection);
+        void inspectWithStrategy(PerformanceInspection, InspectionStrategy);
+        List<PerformanceTestResult> parse();
+    }
+
+    class PerformanceInspectionService {
+    }
+
+    class PerformanceTestResult {
+        String category
+        Object performanceData
+        List<PerformanceInspectionResult> performanceInspectionResultList
+    }
+
+    class PerformanceInspectionResult {
+        String type;
+        File profilingRawResultFile;
+    }
+
+    PerformanceInspector -left..> PerformanceTestSpec
+    PerformanceInspector -up..> PerformanceInspectionResult
+    PerformanceInspector <---right PerformanceInspectionService
+    PerformanceInspector -up..> PerformanceTestResult
+    PerformanceTestResult -right..> PerformanceInspectionResult
+    PerformanceInspectionService -u-|> IPerformanceInspectionService
+}
+
+package "agent" {
+    abstract class TestRunner {
+    }
+}
+
+package "common" {
+    class PerformanceTestManagementService {
+
+    }
+
+    PerformanceInspector <|-- AndroidBatteryInspector
+    PerformanceInspector <|-- AndroidMemoryInfoInspector
+    PerformanceInspector <|-- AndroidMemoryDumpInspector
+    PerformanceInspector <|-- WindowsBatteryInspector
+    PerformanceInspector <|-- WindowsMemoryInspector
+
+    AndroidBatteryInspector --o PerformanceTestManagementService
+    AndroidMemoryInfoInspector --o PerformanceTestManagementService
+    AndroidMemoryDumpInspector --o PerformanceTestManagementService
+    WindowsBatteryInspector --o PerformanceTestManagementService
+    WindowsMemoryInspector --o PerformanceTestManagementService
+    PerformanceTestManagementService --|> IPerformanceInspectionService
+
+    TestRunner o-u- PerformanceTestManagementService
+}
+@enduml
+
+@startuml
+participant PerformanceManager
+participant Runner
+participant PerformanceInspectionService
+participant AndroidMemoryInfoInspector
+participant AndroidBatteryInspector
+participant WindowsBatteryInspector
+
+title Sequence 1: Regularly inspect performance metrics
+
+PerformanceManager -> PerformanceManager: new Inspectors
+activate Runner
+Runner -> PerformanceInspectionService : new PerformanceInspectionService()
+activate PerformanceInspectionService
+Runner -> PerformanceManager: get Inspectors
+PerformanceManager -> Runner: Inspectors
+Runner -> PerformanceInspectionService : add Inspectors
+Runner -> PerformanceInspectionService : startInspectPerformanceTimer(performanceTestSpec, interval)
+PerformanceInspectionService -> AndroidMemoryInfoInspector : initialize
+PerformanceInspectionService -> AndroidBatteryInspector : initialize
+PerformanceInspectionService -> WindowsBatteryInspector : initialize
+PerformanceInspectionService -> AndroidMemoryInfoInspector : inspect
+AndroidMemoryInfoInspector -> PerformanceInspectionService : PerformanceInspectionResult
+PerformanceInspectionService -> AndroidBatteryInspector : inspect
+AndroidBatteryInspector -> PerformanceInspectionService : PerformanceInspectionResult
+PerformanceInspectionService -> WindowsBatteryInspector : inspect
+WindowsBatteryInspector -> PerformanceInspectionService : PerformanceInspectionResult
+Runner -> PerformanceInspectionService : parse
+PerformanceInspectionService -> AndroidMemoryInfoInspector : parse
+AndroidMemoryInfoInspector -> PerformanceInspectionService : PerformanceTestResult
+PerformanceInspectionService -> AndroidBatteryInspector : parse
+AndroidBatteryInspector -> PerformanceInspectionService : PerformanceTestResult
+PerformanceInspectionService -> WindowsBatteryInspector : parse
+WindowsBatteryInspector -> PerformanceInspectionService : PerformanceTestResult
+PerformanceInspectionService -> Runner: List<PerformanceTestResult>
+deactivate PerformanceInspectionService
+deactivate Runner
+@enduml
+
+@startuml
+participant PerformanceManager
+participant Runner
+participant TestCase
+participant PerformanceInspectionService
+participant AndroidMemoryInfoInspector
+participant AndroidBatteryInspector
+participant WindowsBatteryInspector
+participant ThreadParam
+
+title Sequence 2: Trigger performance metrics inspection by test case
+
+PerformanceManager -> PerformanceManager: new Inspectors
+activate Runner
+Runner -> PerformanceInspectionService : new PerformanceInspectionService()
+activate PerformanceInspectionService
+Runner -> PerformanceManager: get Inspectors
+PerformanceManager -> Runner: Inspectors
+Runner -> PerformanceInspectionService : add Inspectors
+Runner -> ThreadParam: init(..., PerformanceInspectionService)
+activate ThreadParam
+Runner -> TestCase : execute
+activate TestCase
+TestCase -> ThreadParam : getPerformanceInspectionService
+ThreadParam -> TestCase : PerformanceInspectionService
+TestCase -> PerformanceInspectionService : initialize
+PerformanceInspectionService -> AndroidMemoryInfoInspector : initialize
+PerformanceInspectionService -> AndroidBatteryInspector : initialize
+PerformanceInspectionService -> WindowsBatteryInspector : initialize
+TestCase -> PerformanceInspectionService : inspect
+PerformanceInspectionService -> AndroidMemoryInfoInspector : inspect
+AndroidMemoryInfoInspector -> PerformanceInspectionService : PerformanceInspectionResult
+PerformanceInspectionService -> AndroidBatteryInspector : inspect
+AndroidBatteryInspector -> PerformanceInspectionService : PerformanceInspectionResult
+PerformanceInspectionService -> WindowsBatteryInspector : inspect
+WindowsBatteryInspector -> PerformanceInspectionService : PerformanceInspectionResult
+PerformanceInspectionService -> TestCase : List<PerformanceInspectionResult>
+TestCase -> Runner: return
+deactivate
+Runner -> PerformanceInspectionService : parse
+PerformanceInspectionService -> AndroidMemoryInfoInspector : parse
+AndroidMemoryInfoInspector -> PerformanceInspectionService : PerformanceTestResult
+PerformanceInspectionService -> AndroidBatteryInspector : parse
+AndroidBatteryInspector -> PerformanceInspectionService : PerformanceTestResult
+PerformanceInspectionService -> WindowsBatteryInspector : parse
+WindowsBatteryInspector -> PerformanceInspectionService : PerformanceTestResult
+PerformanceInspectionService -> Runner: List<PerformanceTestResult>
+deactivate PerformanceInspectionService
+Runner -> ThreadParam: clean
+deactivate ThreadParam
+deactivate Runner
+@enduml

common/src/main/java/com/microsoft/hydralab/common/entity/common/TestRun.java

@@ -4,6 +4,7 @@
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.microsoft.hydralab.ITestRun;
 import com.microsoft.hydralab.common.util.Const;
 import lombok.Data;
 import org.slf4j.Logger;
@@ -19,7 +20,7 @@
 @Entity
 @Table(name = "device_test_task", indexes = {
         @Index(name = "task_id_index", columnList = "test_task_id")})
-public class TestRun implements Serializable {
+public class TestRun implements Serializable, ITestRun {
     //    private static Pattern testResultLine = Pattern.compile("Tests run:\\s+(\\d+),\\s+Failures:\\s+(\\d+)");
     // OK (8 tests)
 //    private static Pattern testResultOkLine = Pattern.compile("OK\\s+\\((\\d+)\\s+tests\\)");

common/src/main/java/com/microsoft/hydralab/performance/PerformanceTestManagementService.java

@@ -0,0 +1,66 @@
+package com.microsoft.hydralab.performance;
+
+import com.microsoft.hydralab.ITestRun;
+import com.microsoft.hydralab.performance.inspectors.AndroidBatteryInfoInspector;
+import org.jetbrains.annotations.NotNull;
+import org.springframework.util.Assert;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class PerformanceTestManagementService implements IPerformanceInspectionService {
+    private final Map<String, PerformanceInspector> performanceInspectorMap = Map.of(
+            PerformanceInspector.INSPECTOR_ANDROID_BATTERY_INFO, new AndroidBatteryInfoInspector()
+    );
+    private final Map<ITestRun, Map<String, PerformanceTestResult>> testRunPerfResultMap = new ConcurrentHashMap<>();
+
+
+    public void initialize() {
+        PerformanceInspectionService.getInstance().swapImplementation(this);
+    }
+    private PerformanceInspector getInspectorByName(String inspectorName) {
+        return performanceInspectorMap.get(inspectorName);
+    }
+    @Override
+    public PerformanceInspectionResult inspect(PerformanceInspection performanceInspection) {
+        String inspector = performanceInspection.inspector;
+        PerformanceInspector performanceInspector = getInspectorByName(inspector);
+        Assert.notNull(performanceInspector, "Found no matched inspector: " + performanceInspection.inspector);
+        ITestRun testRun = getTestRun();
+        File performanceFolder = new File(testRun.getResultFolder(), "performance");
+        Assert.isTrue(performanceFolder.mkdirs(), "performanceInspection.resultFolder.mkdirs() failed in " + performanceFolder.getAbsolutePath());
+        performanceInspection.resultFolder = performanceFolder;
+
+        PerformanceInspectionResult result = performanceInspector.inspect(performanceInspection);
+
+        Map<String, PerformanceTestResult> performanceTestResultMap = testRunPerfResultMap.putIfAbsent(getTestRun(), new HashMap<>());
+        Assert.notNull(performanceTestResultMap, "performanceTestResultMap should not be null ");
+        PerformanceTestResult performanceTestResult = performanceTestResultMap.putIfAbsent(performanceInspection.inspectionKey, createPerformanceTestResult(performanceInspection));
+        Assert.notNull(performanceTestResult, "performanceTestResult should not be null ");
+        performanceTestResult.performanceInspectionResults.add(result);
+
+        return result;
+    }
+
+    @NotNull
+    private static PerformanceTestResult createPerformanceTestResult(PerformanceInspection performanceInspection) {
+        PerformanceTestResult performanceTestResult = new PerformanceTestResult();
+        performanceTestResult.inspector = performanceInspection.inspector;
+        return performanceTestResult;
+    }
+
+    /**
+     * TODO
+     * @return the test run object from TestRunThreadContext
+     */
+    private ITestRun getTestRun() {
+        return null;
+    }
+
+    @Override
+    public void inspectWithStrategy(PerformanceInspection performanceInspection, InspectionStrategy inspectionStrategy) {
+
+    }
+}

common/src/main/java/com/microsoft/hydralab/performance/inspectors/AndroidBatteryInfoInspector.java

@@ -0,0 +1,12 @@
+package com.microsoft.hydralab.performance.inspectors;
+
+import com.microsoft.hydralab.performance.PerformanceInspection;
+import com.microsoft.hydralab.performance.PerformanceInspectionResult;
+import com.microsoft.hydralab.performance.PerformanceInspector;
+
+public class AndroidBatteryInfoInspector implements PerformanceInspector {
+    @Override
+    public PerformanceInspectionResult inspect(PerformanceInspection performanceInspection) {
+        return null;
+    }
+}

sdk/src/main/java/com/microsoft/hydralab/ITestRun.java

@@ -0,0 +1,7 @@
+package com.microsoft.hydralab;
+
+import java.io.File;
+
+public interface ITestRun {
+    File getResultFolder();
+}

sdk/src/main/java/com/microsoft/hydralab/appium/ThreadParam.java

@@ -4,6 +4,9 @@
 
 import java.util.Map;
 
+/**
+ * TODO: rename this to TestRunThreadContext and move this above to package com.microsoft.hydralab
+ */
 public class ThreadParam {
     private static InheritableThreadLocal<AppiumParam> appiumParam = new InheritableThreadLocal<>();
     private static InheritableThreadLocal<Map<String, String>> configMap = new InheritableThreadLocal<>();

sdk/src/main/java/com/microsoft/hydralab/performance/IPerformanceInspectionService.java

@@ -0,0 +1,6 @@
+package com.microsoft.hydralab.performance;
+
+public interface IPerformanceInspectionService {
+    PerformanceInspectionResult inspect(PerformanceInspection performanceInspection);
+    void inspectWithStrategy(PerformanceInspection performanceInspection, InspectionStrategy inspectionStrategy);
+}

sdk/src/main/java/com/microsoft/hydralab/performance/InspectionStrategy.java

@@ -0,0 +1,4 @@
+package com.microsoft.hydralab.performance;
+
+public class InspectionStrategy {
+}

sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspection.java

@@ -0,0 +1,29 @@
+package com.microsoft.hydralab.performance;
+
+import java.io.File;
+
+public class PerformanceInspection {
+    public final String inspector;
+    public final String appId;
+    public final String deviceIdentifier;
+    public final String name;
+    public final String inspectionKey;
+    File resultFolder;
+
+    public PerformanceInspection(String name, String inspector, String appId, String deviceIdentifier) {
+        this.inspector = inspector;
+        this.appId = appId;
+        this.deviceIdentifier = deviceIdentifier;
+        this.name = name;
+        inspectionKey = String.format("%s-%s-%s", appId, deviceIdentifier, inspector);
+    }
+
+    public static PerformanceInspection createAndroidBatteryInfoSpec(String appId, String deviceIdentifier) {
+        return new PerformanceInspection(getNameByParam(PerformanceInspector.INSPECTOR_ANDROID_BATTERY_INFO, appId, deviceIdentifier),
+                PerformanceInspector.INSPECTOR_ANDROID_BATTERY_INFO, appId, deviceIdentifier);
+    }
+
+    private static String getNameByParam(String inspector, String appId, String deviceId) {
+        return String.format("PerfTesting: get %s for %s on %s", inspector, appId, deviceId);
+    }
+}

sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspectionResult.java

@@ -0,0 +1,4 @@
+package com.microsoft.hydralab.performance;
+
+public class PerformanceInspectionResult {
+}

sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspectionService.java

@@ -0,0 +1,35 @@
+package com.microsoft.hydralab.performance;
+
+public enum PerformanceInspectionService implements IPerformanceInspectionService {
+    INSTANCE;
+
+    public static PerformanceInspectionService getInstance() {
+        return INSTANCE;
+    }
+
+    private IPerformanceInspectionService serviceImplementation = new IPerformanceInspectionService() {
+        @Override
+        public PerformanceInspectionResult inspect(PerformanceInspection performanceInspection) {
+            return null;
+        }
+
+        @Override
+        public void inspectWithStrategy(PerformanceInspection performanceInspection, InspectionStrategy inspectionStrategy) {
+
+        }
+    };
+
+    void swapImplementation(IPerformanceInspectionService serviceImplementation) {
+        this.serviceImplementation = serviceImplementation;
+    }
+
+    @Override
+    public PerformanceInspectionResult inspect(PerformanceInspection performanceInspection) {
+        return serviceImplementation.inspect(performanceInspection);
+    }
+
+    @Override
+    public void inspectWithStrategy(PerformanceInspection performanceInspection, InspectionStrategy inspectionStrategy) {
+
+    }
+}

sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspector.java

@@ -0,0 +1,10 @@
+package com.microsoft.hydralab.performance;
+
+public interface PerformanceInspector {
+    String INSPECTOR_ANDROID_MEMORY_DUMP = "AndroidMemoryDump";
+    String INSPECTOR_ANDROID_MEMORY_INFO = "AndroidMemoryInfo";
+    String INSPECTOR_ANDROID_BATTERY_INFO = "AndroidBatteryInfo";
+    String INSPECTOR_WIN_BATTERY = "WindowsBattery";
+    String INSPECTOR_WIN_MEMORY = "WindowsMemory";
+    PerformanceInspectionResult inspect(PerformanceInspection performanceInspection);
+}

sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceTestResult.java

@@ -0,0 +1,20 @@
+package com.microsoft.hydralab.performance;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class PerformanceTestResult {
+    /**
+     * memory: java_heap_pss	java_heap_rss	native_heap_pss	native_heap_rss	code_pss	code_rss	stack_pss	stack_rss
+     * graphics_pss	graphics_rss	private_other_pss	private_other_rss	system_pss	system_rss	unknown_pss	unknown_rss
+     * total_pss	total_rss	total_swap_pss
+     * <p>
+     * battery: CPU	screen	Wake_lock	other	App_usage	Total_usage
+     */
+    public Object resultSummary;
+    /**
+     * TODO: Apply a max size to avoid OOM
+     */
+    public List<PerformanceInspectionResult> performanceInspectionResults = new ArrayList<>();
+    public String inspector;
+}