diff --git a/common/doc/UML/perf_design.puml b/common/doc/UML/perf_design.puml new file mode 100644 index 000000000..0491b72e1 --- /dev/null +++ b/common/doc/UML/perf_design.puml @@ -0,0 +1,164 @@ +@startuml +'https://plantuml.com/class-diagram' + +package "sdk" { + + class PerformanceTestSpec { + -String inspector + -String appId + -String deviceId + -String name + } + interface PerformanceInspector { + void initialize(PerformanceTestSpec performanceTestSpec) + PerformanceInspectionResult inspect(PerformanceTestSpec performanceTestSpec) + PerformanceTestResult parse(List) + } + + interface IPerformanceInspectionService { + void reset(PerformanceInspection); + PerformanceInspectionResult inspect(PerformanceInspection); + void inspectWithStrategy(PerformanceInspection, InspectionStrategy); + List parse(); + } + + class PerformanceInspectionService { + } + + class PerformanceTestResult { + String category + Object performanceData + List performanceInspectionResultList + } + + class PerformanceInspectionResult { + String type; + File profilingRawResultFile; + } + + PerformanceInspector -left..> PerformanceTestSpec + PerformanceInspector -up..> PerformanceInspectionResult + PerformanceInspector <---right PerformanceInspectionService + PerformanceInspector -up..> PerformanceTestResult + PerformanceTestResult -right..> PerformanceInspectionResult + PerformanceInspectionService -u-|> IPerformanceInspectionService +} + +package "agent" { + abstract class TestRunner { + } +} + +package "common" { + class PerformanceTestManagementService { + + } + + PerformanceInspector <|-- AndroidBatteryInspector + PerformanceInspector <|-- AndroidMemoryInfoInspector + PerformanceInspector <|-- AndroidMemoryDumpInspector + PerformanceInspector <|-- WindowsBatteryInspector + PerformanceInspector <|-- WindowsMemoryInspector + + AndroidBatteryInspector --o PerformanceTestManagementService + AndroidMemoryInfoInspector --o PerformanceTestManagementService + AndroidMemoryDumpInspector --o PerformanceTestManagementService + WindowsBatteryInspector --o PerformanceTestManagementService + WindowsMemoryInspector --o PerformanceTestManagementService + PerformanceTestManagementService --|> IPerformanceInspectionService + + TestRunner o-u- PerformanceTestManagementService +} +@enduml + +@startuml +participant PerformanceManager +participant Runner +participant PerformanceInspectionService +participant AndroidMemoryInfoInspector +participant AndroidBatteryInspector +participant WindowsBatteryInspector + +title Sequence 1: Regularly inspect performance metrics + +PerformanceManager -> PerformanceManager: new Inspectors +activate Runner +Runner -> PerformanceInspectionService : new PerformanceInspectionService() +activate PerformanceInspectionService +Runner -> PerformanceManager: get Inspectors +PerformanceManager -> Runner: Inspectors +Runner -> PerformanceInspectionService : add Inspectors +Runner -> PerformanceInspectionService : startInspectPerformanceTimer(performanceTestSpec, interval) +PerformanceInspectionService -> AndroidMemoryInfoInspector : initialize +PerformanceInspectionService -> AndroidBatteryInspector : initialize +PerformanceInspectionService -> WindowsBatteryInspector : initialize +PerformanceInspectionService -> AndroidMemoryInfoInspector : inspect +AndroidMemoryInfoInspector -> PerformanceInspectionService : PerformanceInspectionResult +PerformanceInspectionService -> AndroidBatteryInspector : inspect +AndroidBatteryInspector -> PerformanceInspectionService : PerformanceInspectionResult +PerformanceInspectionService -> WindowsBatteryInspector : inspect +WindowsBatteryInspector -> PerformanceInspectionService : PerformanceInspectionResult +Runner -> PerformanceInspectionService : parse +PerformanceInspectionService -> AndroidMemoryInfoInspector : parse +AndroidMemoryInfoInspector -> PerformanceInspectionService : PerformanceTestResult +PerformanceInspectionService -> AndroidBatteryInspector : parse +AndroidBatteryInspector -> PerformanceInspectionService : PerformanceTestResult +PerformanceInspectionService -> WindowsBatteryInspector : parse +WindowsBatteryInspector -> PerformanceInspectionService : PerformanceTestResult +PerformanceInspectionService -> Runner: List +deactivate PerformanceInspectionService +deactivate Runner +@enduml + +@startuml +participant PerformanceManager +participant Runner +participant TestCase +participant PerformanceInspectionService +participant AndroidMemoryInfoInspector +participant AndroidBatteryInspector +participant WindowsBatteryInspector +participant ThreadParam + +title Sequence 2: Trigger performance metrics inspection by test case + +PerformanceManager -> PerformanceManager: new Inspectors +activate Runner +Runner -> PerformanceInspectionService : new PerformanceInspectionService() +activate PerformanceInspectionService +Runner -> PerformanceManager: get Inspectors +PerformanceManager -> Runner: Inspectors +Runner -> PerformanceInspectionService : add Inspectors +Runner -> ThreadParam: init(..., PerformanceInspectionService) +activate ThreadParam +Runner -> TestCase : execute +activate TestCase +TestCase -> ThreadParam : getPerformanceInspectionService +ThreadParam -> TestCase : PerformanceInspectionService +TestCase -> PerformanceInspectionService : initialize +PerformanceInspectionService -> AndroidMemoryInfoInspector : initialize +PerformanceInspectionService -> AndroidBatteryInspector : initialize +PerformanceInspectionService -> WindowsBatteryInspector : initialize +TestCase -> PerformanceInspectionService : inspect +PerformanceInspectionService -> AndroidMemoryInfoInspector : inspect +AndroidMemoryInfoInspector -> PerformanceInspectionService : PerformanceInspectionResult +PerformanceInspectionService -> AndroidBatteryInspector : inspect +AndroidBatteryInspector -> PerformanceInspectionService : PerformanceInspectionResult +PerformanceInspectionService -> WindowsBatteryInspector : inspect +WindowsBatteryInspector -> PerformanceInspectionService : PerformanceInspectionResult +PerformanceInspectionService -> TestCase : List +TestCase -> Runner: return +deactivate +Runner -> PerformanceInspectionService : parse +PerformanceInspectionService -> AndroidMemoryInfoInspector : parse +AndroidMemoryInfoInspector -> PerformanceInspectionService : PerformanceTestResult +PerformanceInspectionService -> AndroidBatteryInspector : parse +AndroidBatteryInspector -> PerformanceInspectionService : PerformanceTestResult +PerformanceInspectionService -> WindowsBatteryInspector : parse +WindowsBatteryInspector -> PerformanceInspectionService : PerformanceTestResult +PerformanceInspectionService -> Runner: List +deactivate PerformanceInspectionService +Runner -> ThreadParam: clean +deactivate ThreadParam +deactivate Runner +@enduml \ No newline at end of file diff --git a/common/src/main/java/com/microsoft/hydralab/common/entity/common/TestRun.java b/common/src/main/java/com/microsoft/hydralab/common/entity/common/TestRun.java index 611f4a3c9..a1973604a 100644 --- a/common/src/main/java/com/microsoft/hydralab/common/entity/common/TestRun.java +++ b/common/src/main/java/com/microsoft/hydralab/common/entity/common/TestRun.java @@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; +import com.microsoft.hydralab.ITestRun; import com.microsoft.hydralab.common.util.Const; import lombok.Data; import org.slf4j.Logger; @@ -19,7 +20,7 @@ @Entity @Table(name = "device_test_task", indexes = { @Index(name = "task_id_index", columnList = "test_task_id")}) -public class TestRun implements Serializable { +public class TestRun implements Serializable, ITestRun { // private static Pattern testResultLine = Pattern.compile("Tests run:\\s+(\\d+),\\s+Failures:\\s+(\\d+)"); // OK (8 tests) // private static Pattern testResultOkLine = Pattern.compile("OK\\s+\\((\\d+)\\s+tests\\)"); diff --git a/common/src/main/java/com/microsoft/hydralab/performance/PerformanceTestManagementService.java b/common/src/main/java/com/microsoft/hydralab/performance/PerformanceTestManagementService.java new file mode 100644 index 000000000..e12b0b03b --- /dev/null +++ b/common/src/main/java/com/microsoft/hydralab/performance/PerformanceTestManagementService.java @@ -0,0 +1,66 @@ +package com.microsoft.hydralab.performance; + +import com.microsoft.hydralab.ITestRun; +import com.microsoft.hydralab.performance.inspectors.AndroidBatteryInfoInspector; +import org.jetbrains.annotations.NotNull; +import org.springframework.util.Assert; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class PerformanceTestManagementService implements IPerformanceInspectionService { + private final Map performanceInspectorMap = Map.of( + PerformanceInspector.INSPECTOR_ANDROID_BATTERY_INFO, new AndroidBatteryInfoInspector() + ); + private final Map> testRunPerfResultMap = new ConcurrentHashMap<>(); + + + public void initialize() { + PerformanceInspectionService.getInstance().swapImplementation(this); + } + private PerformanceInspector getInspectorByName(String inspectorName) { + return performanceInspectorMap.get(inspectorName); + } + @Override + public PerformanceInspectionResult inspect(PerformanceInspection performanceInspection) { + String inspector = performanceInspection.inspector; + PerformanceInspector performanceInspector = getInspectorByName(inspector); + Assert.notNull(performanceInspector, "Found no matched inspector: " + performanceInspection.inspector); + ITestRun testRun = getTestRun(); + File performanceFolder = new File(testRun.getResultFolder(), "performance"); + Assert.isTrue(performanceFolder.mkdirs(), "performanceInspection.resultFolder.mkdirs() failed in " + performanceFolder.getAbsolutePath()); + performanceInspection.resultFolder = performanceFolder; + + PerformanceInspectionResult result = performanceInspector.inspect(performanceInspection); + + Map performanceTestResultMap = testRunPerfResultMap.putIfAbsent(getTestRun(), new HashMap<>()); + Assert.notNull(performanceTestResultMap, "performanceTestResultMap should not be null "); + PerformanceTestResult performanceTestResult = performanceTestResultMap.putIfAbsent(performanceInspection.inspectionKey, createPerformanceTestResult(performanceInspection)); + Assert.notNull(performanceTestResult, "performanceTestResult should not be null "); + performanceTestResult.performanceInspectionResults.add(result); + + return result; + } + + @NotNull + private static PerformanceTestResult createPerformanceTestResult(PerformanceInspection performanceInspection) { + PerformanceTestResult performanceTestResult = new PerformanceTestResult(); + performanceTestResult.inspector = performanceInspection.inspector; + return performanceTestResult; + } + + /** + * TODO + * @return the test run object from TestRunThreadContext + */ + private ITestRun getTestRun() { + return null; + } + + @Override + public void inspectWithStrategy(PerformanceInspection performanceInspection, InspectionStrategy inspectionStrategy) { + + } +} diff --git a/common/src/main/java/com/microsoft/hydralab/performance/inspectors/AndroidBatteryInfoInspector.java b/common/src/main/java/com/microsoft/hydralab/performance/inspectors/AndroidBatteryInfoInspector.java new file mode 100644 index 000000000..82a2a5b65 --- /dev/null +++ b/common/src/main/java/com/microsoft/hydralab/performance/inspectors/AndroidBatteryInfoInspector.java @@ -0,0 +1,12 @@ +package com.microsoft.hydralab.performance.inspectors; + +import com.microsoft.hydralab.performance.PerformanceInspection; +import com.microsoft.hydralab.performance.PerformanceInspectionResult; +import com.microsoft.hydralab.performance.PerformanceInspector; + +public class AndroidBatteryInfoInspector implements PerformanceInspector { + @Override + public PerformanceInspectionResult inspect(PerformanceInspection performanceInspection) { + return null; + } +} diff --git a/sdk/src/main/java/com/microsoft/hydralab/ITestRun.java b/sdk/src/main/java/com/microsoft/hydralab/ITestRun.java new file mode 100644 index 000000000..e64d7dca1 --- /dev/null +++ b/sdk/src/main/java/com/microsoft/hydralab/ITestRun.java @@ -0,0 +1,7 @@ +package com.microsoft.hydralab; + +import java.io.File; + +public interface ITestRun { + File getResultFolder(); +} diff --git a/sdk/src/main/java/com/microsoft/hydralab/appium/ThreadParam.java b/sdk/src/main/java/com/microsoft/hydralab/appium/ThreadParam.java index 742c21bae..d0f598fc1 100644 --- a/sdk/src/main/java/com/microsoft/hydralab/appium/ThreadParam.java +++ b/sdk/src/main/java/com/microsoft/hydralab/appium/ThreadParam.java @@ -4,6 +4,9 @@ import java.util.Map; +/** + * TODO: rename this to TestRunThreadContext and move this above to package com.microsoft.hydralab + */ public class ThreadParam { private static InheritableThreadLocal appiumParam = new InheritableThreadLocal<>(); private static InheritableThreadLocal> configMap = new InheritableThreadLocal<>(); diff --git a/sdk/src/main/java/com/microsoft/hydralab/performance/IPerformanceInspectionService.java b/sdk/src/main/java/com/microsoft/hydralab/performance/IPerformanceInspectionService.java new file mode 100644 index 000000000..8e12597e2 --- /dev/null +++ b/sdk/src/main/java/com/microsoft/hydralab/performance/IPerformanceInspectionService.java @@ -0,0 +1,6 @@ +package com.microsoft.hydralab.performance; + +public interface IPerformanceInspectionService { + PerformanceInspectionResult inspect(PerformanceInspection performanceInspection); + void inspectWithStrategy(PerformanceInspection performanceInspection, InspectionStrategy inspectionStrategy); +} diff --git a/sdk/src/main/java/com/microsoft/hydralab/performance/InspectionStrategy.java b/sdk/src/main/java/com/microsoft/hydralab/performance/InspectionStrategy.java new file mode 100644 index 000000000..954399f79 --- /dev/null +++ b/sdk/src/main/java/com/microsoft/hydralab/performance/InspectionStrategy.java @@ -0,0 +1,4 @@ +package com.microsoft.hydralab.performance; + +public class InspectionStrategy { +} diff --git a/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspection.java b/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspection.java new file mode 100644 index 000000000..775f04a63 --- /dev/null +++ b/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspection.java @@ -0,0 +1,29 @@ +package com.microsoft.hydralab.performance; + +import java.io.File; + +public class PerformanceInspection { + public final String inspector; + public final String appId; + public final String deviceIdentifier; + public final String name; + public final String inspectionKey; + File resultFolder; + + public PerformanceInspection(String name, String inspector, String appId, String deviceIdentifier) { + this.inspector = inspector; + this.appId = appId; + this.deviceIdentifier = deviceIdentifier; + this.name = name; + inspectionKey = String.format("%s-%s-%s", appId, deviceIdentifier, inspector); + } + + public static PerformanceInspection createAndroidBatteryInfoSpec(String appId, String deviceIdentifier) { + return new PerformanceInspection(getNameByParam(PerformanceInspector.INSPECTOR_ANDROID_BATTERY_INFO, appId, deviceIdentifier), + PerformanceInspector.INSPECTOR_ANDROID_BATTERY_INFO, appId, deviceIdentifier); + } + + private static String getNameByParam(String inspector, String appId, String deviceId) { + return String.format("PerfTesting: get %s for %s on %s", inspector, appId, deviceId); + } +} diff --git a/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspectionResult.java b/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspectionResult.java new file mode 100644 index 000000000..53bfcf125 --- /dev/null +++ b/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspectionResult.java @@ -0,0 +1,4 @@ +package com.microsoft.hydralab.performance; + +public class PerformanceInspectionResult { +} diff --git a/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspectionService.java b/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspectionService.java new file mode 100644 index 000000000..01c887d77 --- /dev/null +++ b/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspectionService.java @@ -0,0 +1,35 @@ +package com.microsoft.hydralab.performance; + +public enum PerformanceInspectionService implements IPerformanceInspectionService { + INSTANCE; + + public static PerformanceInspectionService getInstance() { + return INSTANCE; + } + + private IPerformanceInspectionService serviceImplementation = new IPerformanceInspectionService() { + @Override + public PerformanceInspectionResult inspect(PerformanceInspection performanceInspection) { + return null; + } + + @Override + public void inspectWithStrategy(PerformanceInspection performanceInspection, InspectionStrategy inspectionStrategy) { + + } + }; + + void swapImplementation(IPerformanceInspectionService serviceImplementation) { + this.serviceImplementation = serviceImplementation; + } + + @Override + public PerformanceInspectionResult inspect(PerformanceInspection performanceInspection) { + return serviceImplementation.inspect(performanceInspection); + } + + @Override + public void inspectWithStrategy(PerformanceInspection performanceInspection, InspectionStrategy inspectionStrategy) { + + } +} diff --git a/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspector.java b/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspector.java new file mode 100644 index 000000000..ef4a56511 --- /dev/null +++ b/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspector.java @@ -0,0 +1,10 @@ +package com.microsoft.hydralab.performance; + +public interface PerformanceInspector { + String INSPECTOR_ANDROID_MEMORY_DUMP = "AndroidMemoryDump"; + String INSPECTOR_ANDROID_MEMORY_INFO = "AndroidMemoryInfo"; + String INSPECTOR_ANDROID_BATTERY_INFO = "AndroidBatteryInfo"; + String INSPECTOR_WIN_BATTERY = "WindowsBattery"; + String INSPECTOR_WIN_MEMORY = "WindowsMemory"; + PerformanceInspectionResult inspect(PerformanceInspection performanceInspection); +} diff --git a/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceTestResult.java b/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceTestResult.java new file mode 100644 index 000000000..ec97de15d --- /dev/null +++ b/sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceTestResult.java @@ -0,0 +1,20 @@ +package com.microsoft.hydralab.performance; + +import java.util.ArrayList; +import java.util.List; + +public class PerformanceTestResult { + /** + * memory: java_heap_pss java_heap_rss native_heap_pss native_heap_rss code_pss code_rss stack_pss stack_rss + * graphics_pss graphics_rss private_other_pss private_other_rss system_pss system_rss unknown_pss unknown_rss + * total_pss total_rss total_swap_pss + *

+ * battery: CPU screen Wake_lock other App_usage Total_usage + */ + public Object resultSummary; + /** + * TODO: Apply a max size to avoid OOM + */ + public List performanceInspectionResults = new ArrayList<>(); + public String inspector; +} diff --git a/sdk/src/test/java/com/microsoft/hydralab/performanc/PerformanceInspectionServiceTest.java b/sdk/src/test/java/com/microsoft/hydralab/performanc/PerformanceInspectionServiceTest.java new file mode 100644 index 000000000..34f6734cd --- /dev/null +++ b/sdk/src/test/java/com/microsoft/hydralab/performanc/PerformanceInspectionServiceTest.java @@ -0,0 +1,17 @@ +package com.microsoft.hydralab.performanc; + +import com.microsoft.hydralab.performance.PerformanceInspection; +import com.microsoft.hydralab.performance.PerformanceInspectionResult; +import com.microsoft.hydralab.performance.PerformanceInspectionService; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class PerformanceInspectionServiceTest { + @Test + public void serviceWithInit_Inspect_ReturnNull() { + PerformanceInspectionResult inspectionResult = + PerformanceInspectionService.getInstance().inspect( + PerformanceInspection.createAndroidBatteryInfoSpec("appId", "deviceIdentifier")); + Assertions.assertNull(inspectionResult); + } +}