Skip to content

Commit

Permalink
[Perf] Add strategy for performance test (#279)
Browse files Browse the repository at this point in the history 
* [Perf] Add strategy for performance test

* Update test_task_spec_design

* Rename IPerformanceListener.java -> PerformanceTestListener

* Resolve comments

* Update agent/doc/UML/performance_design.puml

Co-authored-by: Nathan Bu <370788475@qq.com>

* Apply suggestions for performance test design UML

Co-authored-by: Nathan Bu <370788475@qq.com>

* Refine performance_test_design.puml

* Apply suggestions for PerformanceTestManagementService

Co-authored-by: Nathan Bu <370788475@qq.com>

* Apply suggestions from code review

Co-authored-by: Nathan Bu <370788475@qq.com>

* Update perf design uml
Resolve comments
Fix schedule bug

* Add UT for PerformanceTestManagementService and AndroidBatteryInfoResultParser

* Add PerformanceTestManagementServiceTest

---------

Co-authored-by: Nathan Bu <370788475@qq.com>
  • Loading branch information
@taoran6 @hydraxman
taoran6 and hydraxman committed Feb 23, 2023
1 parent a81cf1d commit 4b5e048
Show file tree
Hide file tree
Showing 37 changed files with 1,191 additions and 88 deletions.
110 changes: 110 additions & 0 deletions agent/doc/UML/performance_test_design.puml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@

@startmindmap performance_test_inspection_entrypoints
* Performance Test
** Define the strategy in TestTask
*** LifeCycle
****:
TEST_STARTED
TEST_FINISHED
TEST_FAILED
TEST_RUN_STARTED
TEST_RUN_FINISHED
;
*** Schedule
** Call inspect() method in a test case
*** Hydra SDK
*** T2C
*** Espresso
@endmindmap
@startjson taps_to_cases_json_with_performance_test_actions
{
"drivers": [
{
"id": "13211FDD400183",
"platform": "android",
' Deprecated Name : init
"setUp": {
"launcherApp": ""
}
' Can also add tearDown here to match the device action design
}
],
' deprecated name: cases
"actions": [
{
"index": 0,
"driverId": "13211FDD400183",
"action": {
"actionType": "inspection",
"arguments": {
"inspectorType": "INSPECTOR_ANDROID_MEMORY_INFO",
"targetApp": "com.microsoft.appmanager",
"description": "Start LTW",
"isReset": false
}
},
"isOptional": false
},
{
"index": 1,
"driverId": "13211FDD400183",
"action": {
"actionType": "inspection",
"arguments": {
"inspectorType": "INSPECTOR_ANDROID_BATTERY_INFO",
"targetApp ": "com.microsoft.appmanager",
"description ": "Start LTW",
"isReset": true
}
},
"isOptional": false
}
]
}
@endjson

@startuml perf_test_class_design
class InspectionStrategy {

+ StrategyType strategyType;

// if the strategyType == TEST_LIFECYCLE
+ List<WhenType> when;

// if the strategyType == TEST_SCHEDULE
+ long interval;
+ TimeUnit intervalUnit;
}

interface PerformanceTestListener {
void testStarted();
void testFinished();
void testFailure();
void testRunStarted();
void testRunFinished();
}

class PerformanceTestManagementService {
Map<String, List<ScheduledFuture<?>>> inspectPerformanceTimerMap
Map<String, List<InspectionStrategy>> testLifeCycleStrategyMap
}

PerformanceTestManagementService -> InspectionStrategy
PerformanceTestManagementService -u-|> PerformanceTestListener

PerformanceTestManagementService -d-o TestRunner
AppiumListener -d-> PerformanceTestListener
Junit5Listener -d-> PerformanceTestListener
EspressoTestInfoProcessorListener -d-> PerformanceTestListener
@enduml

@startuml espresso_performance_test_design
class EspressoPerformanceParser {
void processNewLines()
}
EspressoPerformanceParser --|> InstrumentationResultParser
TestRunner <|-- EspressoRunner
EspressoRunner -> EspressoPerformanceParser
EspressoPerformanceParser -> PerformanceInspectionService
@enduml

29 changes: 28 additions & 1 deletion agent/doc/UML/test_runner_design.puml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,34 @@ deactivate TaskCompletion
"isReset": false
},
"frameworkType": "JUnit4",
"testRunnerName": "androidx.test.runner.AndroidJUnitRunner"
"testRunnerName": "androidx.test.runner.AndroidJUnitRunner",
"inspectionStrategies": [
{
"strategyType": "TEST_SCHEDULE",
"interval": 300,
"intervalUnit": "MILLISECONDS",
"inspection": {
"inspectorType": "INSPECTOR_ANDROID_BATTERY_INFO",
"appId": "com.microsoft.appmanager",
"description": "test schedule"
}
},
{
"strategyType": "TEST_LIFECYCLE",
"when": [
"TEST_STARTED",
"TEST_SUCCESS",
"TEST_FAILURE",
"TEST_RUN_STARTED",
"TEST_RUN_FINISHED"
],
"inspection": {
"inspectorType": "INSPECTOR_ANDROID_BATTERY_INFO",
"appId": "com.microsoft.appmanager",
"description": "test schedule"
}
}
]
}
@endjson

Expand Down
46 changes: 32 additions & 14 deletions agent/src/main/java/com/microsoft/hydralab/agent/config/TestRunnerConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import com.microsoft.hydralab.common.entity.common.TestTask;
import com.microsoft.hydralab.common.management.DeviceManager;
import com.microsoft.hydralab.common.util.ADBOperateUtil;
import com.microsoft.hydralab.performance.PerformanceTestManagementService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
Expand All @@ -39,38 +40,55 @@ public class TestRunnerConfig {
);

@Bean
public EspressoRunner espressoRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService, ADBOperateUtil adbOperateUtil) {
return new EspressoRunner(deviceManager, testTaskEngineService, adbOperateUtil);
public PerformanceTestManagementService performanceTestManagementService() {
PerformanceTestManagementService performanceTestManagementService = new PerformanceTestManagementService();
performanceTestManagementService.initialize();
return performanceTestManagementService;
}

@Bean
public AdbMonkeyRunner adbMonkeyRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService, ADBOperateUtil adbOperateUtil) {
return new AdbMonkeyRunner(deviceManager, testTaskEngineService, adbOperateUtil);
public EspressoRunner espressoRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService,
PerformanceTestManagementService performanceTestManagementService,
ADBOperateUtil adbOperateUtil) {
return new EspressoRunner(deviceManager, testTaskEngineService, performanceTestManagementService, adbOperateUtil);
}

@Bean
public AppiumMonkeyRunner appiumMonkeyRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService) {
return new AppiumMonkeyRunner(deviceManager, testTaskEngineService);
public AdbMonkeyRunner adbMonkeyRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService,
PerformanceTestManagementService performanceTestManagementService,
ADBOperateUtil adbOperateUtil) {
return new AdbMonkeyRunner(deviceManager, testTaskEngineService, performanceTestManagementService, adbOperateUtil);
}

@Bean
public AppiumRunner appiumRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService) {
return new AppiumRunner(deviceManager, testTaskEngineService);
public AppiumMonkeyRunner appiumMonkeyRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService,
PerformanceTestManagementService performanceTestManagementService) {
return new AppiumMonkeyRunner(deviceManager, testTaskEngineService, performanceTestManagementService);
}

@Bean
public AppiumCrossRunner appiumCrossRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService) {
return new AppiumCrossRunner(deviceManager, testTaskEngineService, agentName);
public AppiumRunner appiumRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService,
PerformanceTestManagementService performanceTestManagementService) {
return new AppiumRunner(deviceManager, testTaskEngineService, performanceTestManagementService);
}

@Bean
public SmartRunner smartRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService, SmartTestUtil smartTestUtil) {
return new SmartRunner(deviceManager, testTaskEngineService, smartTestUtil);
public AppiumCrossRunner appiumCrossRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService,
PerformanceTestManagementService performanceTestManagementService) {
return new AppiumCrossRunner(deviceManager, testTaskEngineService, performanceTestManagementService, agentName);
}

@Bean
public T2CRunner t2cRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService) {
return new T2CRunner(deviceManager, testTaskEngineService, agentName);
public SmartRunner smartRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService,
PerformanceTestManagementService performanceTestManagementService,
SmartTestUtil smartTestUtil) {
return new SmartRunner(deviceManager, testTaskEngineService, performanceTestManagementService, smartTestUtil);
}

@Bean
public T2CRunner t2cRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService,
PerformanceTestManagementService performanceTestManagementService) {
return new T2CRunner(deviceManager, testTaskEngineService, performanceTestManagementService, agentName);
}

@ConfigurationProperties(prefix = "app.device-script.commands")
Expand Down
19 changes: 17 additions & 2 deletions agent/src/main/java/com/microsoft/hydralab/agent/runner/TestRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
import com.microsoft.hydralab.common.util.LogUtils;
import com.microsoft.hydralab.common.util.ThreadPoolUtil;
import com.microsoft.hydralab.common.util.ThreadUtils;
import com.microsoft.hydralab.performance.InspectionStrategy;
import com.microsoft.hydralab.performance.PerformanceTestManagementService;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -29,12 +31,15 @@ public abstract class TestRunner {
protected final Logger log = LoggerFactory.getLogger(DeviceManager.class);
protected final DeviceManager deviceManager;
protected final TestTaskRunCallback testTaskRunCallback;
protected final PerformanceTestManagementService performanceTestManagementService;
protected final XmlBuilder xmlBuilder = new XmlBuilder();
protected final ActionExecutor actionExecutor = new ActionExecutor();

public TestRunner(DeviceManager deviceManager, TestTaskRunCallback testTaskRunCallback) {
public TestRunner(DeviceManager deviceManager, TestTaskRunCallback testTaskRunCallback,
PerformanceTestManagementService performanceTestManagementService) {
this.deviceManager = deviceManager;
this.testTaskRunCallback = testTaskRunCallback;
this.performanceTestManagementService = performanceTestManagementService;
}

public void runTestOnDevice(TestTask testTask, DeviceInfo deviceInfo, Logger logger) {
Expand All @@ -52,7 +57,6 @@ public void runTestOnDevice(TestTask testTask, DeviceInfo deviceInfo, Logger log
testRun.getLogger().error(deviceInfo.getSerialNum() + ": " + e.getMessage(), e);
saveErrorSummary(testRun, e);
} finally {
//TODO: tearDown for performance testing. Android battery: adb shell dumpsys battery reset
tearDown(deviceInfo, testTask, testRun);
}
}
Expand Down Expand Up @@ -140,11 +144,22 @@ protected void setUp(DeviceInfo deviceInfo, TestTask testTask, TestRun testRun)

checkTestTaskCancel(testTask);
deviceManager.getScreenShot(deviceInfo, testRun.getLogger());

if (performanceTestManagementService != null && testTask.getInspectionStrategies() != null) {
for (InspectionStrategy strategy : testTask.getInspectionStrategies()) {
performanceTestManagementService.inspectWithStrategy(strategy);
}
}
}

protected abstract void run(DeviceInfo deviceInfo, TestTask testTask, TestRun testRun) throws Exception;

protected void tearDown(DeviceInfo deviceInfo, TestTask testTask, TestRun testRun) {
// stop performance test
if (performanceTestManagementService != null) {
performanceTestManagementService.testTearDown(deviceInfo, log);
}

//execute actions
if (testTask.getDeviceActions() != null) {
testRun.getLogger().info("Start executing tearDown actions.");
Expand Down
6 changes: 4 additions & 2 deletions agent/src/main/java/com/microsoft/hydralab/agent/runner/appium/AppiumCrossRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@
import com.microsoft.hydralab.common.entity.common.TestRun;
import com.microsoft.hydralab.common.entity.common.TestTask;
import com.microsoft.hydralab.common.management.DeviceManager;
import com.microsoft.hydralab.performance.PerformanceTestManagementService;
import org.slf4j.Logger;

public class AppiumCrossRunner extends AppiumRunner {
String agentName;

public AppiumCrossRunner(DeviceManager deviceManager, TestTaskRunCallback testTaskRunCallback, String agentName) {
super(deviceManager, testTaskRunCallback);
public AppiumCrossRunner(DeviceManager deviceManager, TestTaskRunCallback testTaskRunCallback,
PerformanceTestManagementService performanceTestManagementService, String agentName) {
super(deviceManager, testTaskRunCallback, performanceTestManagementService);
this.agentName = agentName;
}

Expand Down
11 changes: 10 additions & 1 deletion agent/src/main/java/com/microsoft/hydralab/agent/runner/appium/AppiumListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import com.microsoft.hydralab.common.logger.LogCollector;
import com.microsoft.hydralab.common.management.DeviceManager;
import com.microsoft.hydralab.common.screen.ScreenRecorder;
import com.microsoft.hydralab.performance.PerformanceTestListener;
import org.junit.runner.Description;
import org.junit.runner.Result;
import org.junit.runner.notification.Failure;
Expand All @@ -31,6 +32,7 @@ public class AppiumListener extends RunListener {
private final AnimatedGifEncoder e = new AnimatedGifEncoder();
private final String pkgName;
DeviceManager deviceManager;
private final PerformanceTestListener performanceTestListener;
private long recordingStartTimeMillis;
private int index;
private File gifFile;
Expand All @@ -43,12 +45,14 @@ public class AppiumListener extends RunListener {
private int currentTestIndex = 0;


public AppiumListener(DeviceManager deviceManager, DeviceInfo deviceInfo, TestRun testRun, String pkgName, Logger logger) {
public AppiumListener(DeviceManager deviceManager, DeviceInfo deviceInfo, TestRun testRun, String pkgName,
PerformanceTestListener performanceTestListener, Logger logger) {
this.deviceManager = deviceManager;
this.deviceInfo = deviceInfo;
this.testRun = testRun;
this.logger = logger;
this.pkgName = pkgName;
this.performanceTestListener = performanceTestListener;
logcatCollector = deviceManager.getLogCollector(deviceInfo, pkgName, testRun, logger);
deviceScreenRecorder = deviceManager.getScreenRecorder(deviceInfo, testRun.getResultFolder(), logger);
}
Expand Down Expand Up @@ -107,6 +111,7 @@ public void testRunStarted(Description description) {
testRun.addNewTimeTag("testRunStarted", System.currentTimeMillis() - recordingStartTimeMillis);
deviceInfo.setRunningTestName(runName.substring(runName.lastIndexOf('.') + 1) + ".testRunStarted");
logEnter(runName, description.testCount());
performanceTestListener.testRunStarted();
}

@Override
Expand Down Expand Up @@ -159,6 +164,7 @@ public void testStarted(Description description) {
ioException.printStackTrace();
}
}), logger);
performanceTestListener.testStarted(ongoingTestUnit.getTitle());
}

@Override
Expand All @@ -167,6 +173,7 @@ public void testFailure(Failure failure) {
logEnter("testFailed", testDisplayName, failure.getTrace());
ongoingTestUnit.setStack(failure.getTrace());
ongoingTestUnit.setStatusCode(AndroidTestUnit.StatusCodes.FAILURE);
performanceTestListener.testFailure(ongoingTestUnit.getTitle());
testRun.addNewTimeTag(ongoingTestUnit.getTitle() + ".fail", System.currentTimeMillis() - recordingStartTimeMillis);
testRun.oneMoreFailure();
}
Expand Down Expand Up @@ -197,6 +204,7 @@ public void testFinished(Description description) {
) {
ongoingTestUnit.setStatusCode(AndroidTestUnit.StatusCodes.OK);
ongoingTestUnit.setSuccess(true);
performanceTestListener.testSuccess(ongoingTestUnit.getTitle());
}
ongoingTestUnit.setEndTimeMillis(System.currentTimeMillis());
ongoingTestUnit.setRelEndTimeInVideo(ongoingTestUnit.getEndTimeMillis() - recordingStartTimeMillis);
Expand Down Expand Up @@ -230,6 +238,7 @@ public void testRunFinished(Result result) {
}

}
performanceTestListener.testRunFinished();

logEnter("testRunEnded", elapsedTime, Thread.currentThread().getName());
synchronized (this) {
Expand Down

0 comments on commit 4b5e048

Please sign in to comment.