Merge a5f90d7 into fbbbdbc

AndroidRunner/Adb.py

@@ -1,5 +1,6 @@
 import logging
 import os.path as op
+from time import sleep
 
 from .pyand import ADB
 
@@ -35,7 +36,7 @@ def connect(device_id):
         raise ConnectionError('No devices are connected')
     logger.debug('Device list:\n%s' % device_list)
     if device_id not in list(device_list.values()):
-        raise ConnectionError('%s: Device can not connected' % device_id)
+        raise ConnectionError('%s: Device not recognized' % device_id)
 
 
 def shell_su(device_id, cmd):
@@ -139,3 +140,14 @@ def logcat(device_id, regex=None):
         params += ' -e %s' % regex
     adb.set_target_by_name(device_id)
     return adb.get_logcat(lcfilter=params)
+
+
+def reset(cmd):
+    if cmd:
+        logger.info('Shutting down adb...')
+        sleep(1)
+        adb.kill_server()
+        sleep(2)
+        logger.info('Restarting adb...')
+        adb.get_devices()
+        sleep(10)

AndroidRunner/Experiment.py

@@ -5,6 +5,7 @@
 from threading import Thread
 
 from . import Tests
+from . import Adb
 import paths
 from .Devices import Devices
 from .Profilers import Profilers
@@ -19,6 +20,7 @@ def __init__(self, config, progress, restart):
         self.progress = progress
         self.basedir = None
         self.random = config.get('randomization', False)
+        Tests.is_valid_option(self.random, valid_options=[True, False])
         if 'devices' not in config:
             raise ConfigError('"device" is required in the configuration')
         adb_path = config.get('adb_path', 'adb')
@@ -28,6 +30,8 @@ def __init__(self, config, progress, restart):
         self.profilers = Profilers(config.get('profilers', {}))
         monkeyrunner_path = config.get('monkeyrunner_path', 'monkeyrunner')
         self.scripts = Scripts(config.get('scripts', {}), monkeyrunner_path=monkeyrunner_path)
+        self.reset_adb_among_runs = config.get('reset_adb_among_runs', False)
+        Tests.is_valid_option(self.reset_adb_among_runs, valid_options=[True, False])
         self.time_between_run = Tests.is_integer(config.get('time_between_run', 0))
         Tests.check_dependencies(self.devices, self.profilers.dependencies())
         self.output_root = paths.OUTPUT_DIR
@@ -208,6 +212,7 @@ def after_run(self, device, path, run, *args, **kwargs):
         """Hook executed after a run"""
         self.scripts.run('after_run', device, *args, **kwargs)
         self.profilers.collect_results(device)
+        Adb.reset(self.reset_adb_among_runs)
         self.logger.debug('Sleeping for %s milliseconds' % self.time_between_run)
         time.sleep(self.time_between_run / 1000.0)
 

AndroidRunner/Tests.py

@@ -24,3 +24,10 @@ def check_dependencies(devices, dependencies):
             for name in not_installed_apps:
                 logging.error('%s: Required package %s is not installed' % (device.id, name))
             raise ConfigError('Required packages %s are not installed on device %s' % (not_installed_apps, device.id))
+
+def is_valid_option(cmd, valid_options):
+    if cmd:
+        match = [x for x in valid_options if x == cmd]
+        if len(match) != 1:
+            raise ConfigError("'%s' not recognized.  Use one of: %s" % (cmd, valid_options))
+    return cmd

README.md

@@ -21,7 +21,7 @@ Additionally, the following are also required for the Batterystats method:
 
 Note: It is important that monkeyrunner shares the same adb the experiment is using. Otherwise, there will be an adb restart and output may be tainted by the notification.
 
-Note 2: You can specifiy the path to ADB and/or Monkeyrunner in the experiment configuration. See the Experiment Configuration section below.
+Note 2: You can specifiy the path to adb and/or Monkeyrunner in the experiment configuration. See the Experiment Configuration section below.
 
 Note 3: To check whether the the device is able to report on the `idle` and `frequency` states of the CPU, you can run the command `python systrace.py -l` and ensure both categories are listed among the supported categories.
 
@@ -34,13 +34,13 @@ Example configuration files can be found in the subdirectories of the `example`
 
 ## Structure
 ### devices.json
-A JSON config that maps devices names to their ADB ids for easy reference in config files.
+A JSON config that maps devices names to their adb ids for easy reference in config files.
 
 ### Experiment Configuration
 Below is a reference to the fields for the experiment configuration. It is not always updated.
 
 **adb_path** *string*
-Path to ADB. Example path: `/opt/platform-tools/adb`
+Path to adb. Example path: `/opt/platform-tools/adb`
 
 **monkeyrunner_path** *string*
 Path to Monkeyrunner. Example path: `/opt/platform-tools/bin/monkeyrunner`
@@ -76,10 +76,13 @@ Number of times each experiment is run.
 Random order of run execution. Default is *false*.
 
 **duration** *positive integer*
-The duration of each run in milliseconds, default is 0. Setting a too short duration may lead to missing results when running native experiments, adviced is to set a higher duration time if unexpected results appear.
+The duration of each run in milliseconds, default is 0. Setting a too short duration may lead to missing results when running native experiments, it is advised to set a higher duration time if unexpected results appear.
+
+**reset_adb_among_runs** *boolean*
+Restarts the adb connection after each run.  Default is *false*.  Recommended to run Android Runner as a privileged user to avoid potential issues with adb device authorizaton.
 
 **time_between_run** *positive integer*
-The time that the framework waits between 2 succesive experiment runs. Default is 0.
+The time that the framework waits between 2 successive experiment runs. Default is 0.
 
 **devices** *JSON*
 A JSON object to describe the devices to be used and their arguments. Below are several examples:
@@ -108,10 +111,10 @@ A JSON object to describe the devices to be used and their arguments. Below are
 ```
 Note that the last two examples result in the same behaviour.
 
-The root_disable_charging option specifies if the devices needs to be root charging disabled by writing the charging_disabled_value to the usb_charging_disabled_file. Different devices have different values for the charging_disabled_value and usb_charging_disabled_file, so be carefull when using this feature. Also keep an eye out on the battery percantage when ussing this feature. If the battery dies when the charging is root disabled, it becomes impossible to charge the device via USB. 
+The root_disable_charging option specifies if the devices needs to be root charging disabled by writing the charging_disabled_value to the usb_charging_disabled_file. Different devices have different values for the charging_disabled_value and usb_charging_disabled_file, so be careful when using this feature. Also keep an eye out on the battery percentage when using this feature. If the battery dies when the charging is root disabled, it becomes impossible to charge the device via USB.
 
 **WARNING:** Always check the battery settings of the device for the charging status of the device after using root disable charging.
-If the device isn't charging after the experiment is finished, reset the charging file yourself via ADB SU command line using:
+If the device isn't charging after the experiment is finished, reset the charging file yourself via adb su command line using:
 ```shell
 adb su -c 'echo <charging enabled value> > <usb_charging_disabled_file>'
 ```
@@ -173,7 +176,7 @@ A JSON object to describe the profilers to be used and their arguments. Below ar
     }
   }
 ```
-The garbage collection (GC) plugin gathers and counts GC log statements by searching in ADB's logcat for logs that meet the format of a GC call as described [here](https://dzone.com/articles/understanding-android-gc-logs).
+The garbage collection (GC) plugin gathers and counts GC log statements by searching in adb's logcat for logs that meet the format of a GC call as described [here](https://dzone.com/articles/understanding-android-gc-logs).
 The default subject aggregation lists the counted GC calls in a single file for easy further processing.
 ```json
   "profilers": {
@@ -183,11 +186,11 @@ The default subject aggregation lists the counted GC calls in a single file for
     }
   }
 ```
-The frame times plugin gathers unique frame rendering durations (in nanoseconds) by utilizing `dumpsys gfxinfo framestats` and counts the amount of delayed frames that occurred following the 16ms threshold [defined by Google](https://developer.android.com/training/testing/performance). 
+The frame times plugin gathers unique frame rendering durations (in nanoseconds) by utilizing `dumpsys gfxinfo framestats` and counts the amount of delayed frames that occurred following the 16ms threshold [defined by Google](https://developer.android.com/training/testing/performance).
 The sample interval is configurable but advised to keep under 120 seconds as the framestats command returns only data from frames rendered in the past 120 seconds as described [here](https://developer.android.com/training/testing/performance).
 Shorter sample intervals will not cause duplication in the frames gathered as only unique frames are kept.
 The default subject aggregation consists of combining both the frametimes as the delayed frames count in single files for easy further processing.
- 
+
 **subject_aggregation** *string*
 Specify which subject aggregation to use. The default is the subject aggregation provided by the profiler. If a user specified aggregation script is used then the script should contain a ```bash main(dummy, data_dir)``` method, as this method is used as the entry point to the script.
 
@@ -229,7 +232,7 @@ Below are the supported types:
   executes after a run completes
 - after_experiment
   executes once after the last run
-  
+
 Instead of a path to string it is also possible to provide a JSON object in the following form:
 ```js
     "interaction": [
@@ -245,7 +248,7 @@ Within the JSON object you can use "type" to "python2", "monkeyrunner" or, "monk
 
 ## Plugin profilers
 It is possible to write your own profiler and use this with Android runner. To do so write your profiler in such a way
-that it uses [this profiler.py class](ExperimentRunner/Plugins/Profiler.py) as parent class. The device object that is mentioned within the profiler.py class is based on the device.py of this repo. To see what can be done with this object, see the source code [here](ExperimentRunner/Device.py).
+that it uses [this profiler.py class](AndroidRunner/Plugins/Profiler.py) as parent class. The device object that is mentioned within the profiler.py class is based on the device.py of this repo. To see what can be done with this object, see the source code [here](AndroidRunner/Device.py).
 
 You can use your own profiler in the same way as the default profilers, you just need to make sure that:
 - The profiler name is the same as your python file and class name.

examples/web/config.json

@@ -4,7 +4,7 @@
     "j7duo2": {}
   },
   "repetitions": 1,
-  "randomization": "False",
+  "randomization": false,
   "browsers": ["firefox"],
   "paths": [
     "https://google.com/",

examples/web/config_root_charging_disabled.json

@@ -8,7 +8,7 @@
     }
    },
   "repetitions": 1,
-  "randomization": "False",
+  "randomization": false,
   "browsers": ["firefox"],
   "paths": [
     "https://google.com/",

tests/unit/test_devices.py

@@ -901,3 +901,17 @@ def test_logcat_with_regex(self):
         expected_calls = [call.set_target_by_name(device_id),
                           call.get_logcat(lcfilter='-d -e {}'.format(test_regex))]
         assert mock_adb.mock_calls == expected_calls
+
+    def test_reset_true(self):
+        Adb.adb = Mock()
+        cmd = True
+        Adb.reset(cmd)
+        expected_calls = [call.kill_server(), call.get_devices()]
+        assert Adb.adb.mock_calls == expected_calls
+
+    def test_reset_false(self):
+        Adb.adb = Mock()
+        cmd = False
+        Adb.reset(cmd)
+        expected_calls = []
+        assert Adb.adb.mock_calls == expected_calls

tests/unit/test_experiment.py

@@ -32,6 +32,7 @@ def test_config(self):
         config['profilers'] = {'fake': {'config1': 1, 'config2': 2}}
         config['monkeyrunner_path'] = 'monkey_path'
         config['scripts'] = {'script1': 'path/to/1'}
+        config['reset_adb_among_runs'] = True
         config['time_between_run'] = 10
         return config
 
@@ -66,6 +67,7 @@ def test_init_only_device_config_no_restart(self, mock_devices, mock_test, mock_
         assert experiment.paths == []
         assert isinstance(experiment.profilers, Profilers)
         assert isinstance(experiment.scripts, Scripts)
+        assert experiment.reset_adb_among_runs is False
         assert experiment.time_between_run == 0
         assert experiment.output_root == paths.OUTPUT_DIR
         assert experiment.result_file_structure is None
@@ -90,6 +92,7 @@ def test_init_only_device_config_restart(self, mock_devices, mock_devices_itter,
         assert experiment.paths == []
         assert isinstance(experiment.profilers, Profilers)
         assert isinstance(experiment.scripts, Scripts)
+        assert experiment.reset_adb_among_runs is False
         assert experiment.time_between_run == 0
         assert experiment.output_root == paths.OUTPUT_DIR
         assert experiment.result_file_structure is None
@@ -122,6 +125,7 @@ def test_init_full_config_no_restart(self, mock_devices, mock_test, mock_profile
         assert experiment.paths == ['test/paths/1', 'test/paths/2']
         assert 'Profilers()' in str(experiment.profilers)
         assert isinstance(experiment.scripts, Scripts)
+        assert experiment.reset_adb_among_runs is True
         assert experiment.time_between_run == 10
         assert experiment.output_root == paths.OUTPUT_DIR
         assert experiment.result_file_structure is None
@@ -209,7 +213,7 @@ def test_walk_to_list(self, default_experiment, tmpdir):
         for item in walk_list:
             for i, file in enumerate(files):
                 flags[i] |= file in item
-        
+
         assert(all(flags))
 
     def test_check_result_files_correct(self, default_experiment, tmpdir):
@@ -495,25 +499,29 @@ def test_before_close(self, script_run, default_experiment):
         script_run.assert_called_once_with('before_close', mock_device, 123, current_activity)
 
     @patch('time.sleep')
+    @patch('AndroidRunner.Adb.reset')
     @patch('AndroidRunner.Profilers.Profilers.collect_results')
     @patch('AndroidRunner.Scripts.Scripts.run')
-    def test_after_run(self, script_run, collect_results, sleep, default_experiment):
+    def test_after_run(self, script_run, collect_results, reset, sleep, default_experiment):
         args = (1, 2, 3)
         kwargs = {'arg1': 1, 'arg2': 2}
         mock_device = Mock()
         path = 'test/path'
         run = 1234566789
+        default_experiment.reset_adb_among_runs = True
         default_experiment.time_between_run = 2000
         mock_manager = Mock()
         mock_manager.attach_mock(script_run, "script_run_managed")
         mock_manager.attach_mock(collect_results, "collect_results_managed")
+        mock_manager.attach_mock(reset, "reset_managed")
         mock_manager.attach_mock(sleep, "sleep_managed")
-
         default_experiment.after_run(mock_device, path, run, *args, **kwargs)
 
         expected_calls = [call.script_run_managed('after_run', mock_device, *args, **kwargs),
                           call.collect_results_managed(mock_device),
-                          call.sleep_managed(2)]
+                          call.reset_managed(True),
+                          call.sleep_managed(2)
+                         ]
         assert mock_manager.mock_calls == expected_calls
 
     def test_after_last_run(self, default_experiment):

tests/unit/test_utils.py

@@ -146,6 +146,25 @@ def test_is_integer_too_small(self):
     def test_is_integer_succes(self):
         assert Tests.is_integer(10) == 10
 
+    def test_cmd_not_valid(self):
+        with pytest.raises(util.ConfigError) as except_result:
+            Tests.is_valid_option("r", ["restart", "abc"])
+        assert "'r' not recognized.  Use one of: ['restart', 'abc']" in str(except_result.value)
+
+    def test_cmd_truthy(self):
+        with pytest.raises(util.ConfigError) as except_result:
+            Tests.is_valid_option("True", [False, True])
+        assert "'True' not recognized.  Use one of: [False, True]" in str(except_result.value)
+
+    def test_more_than_one_cmd(self):
+        with pytest.raises(util.ConfigError) as except_result:
+            Tests.is_valid_option("restart abc", ["restart", "abc"])
+        assert "'restart abc' not recognized.  Use one of: ['restart', 'abc']" in str(except_result.value)
+
+    def test_cmd_is_valid(self):
+        test_command = "foo"
+        assert Tests.is_valid_option(test_command, ["bar","foo"]) == test_command
+
     def test_is_string_fail(self):
         with pytest.raises(util.ConfigError) as except_result:
             Tests.is_string(list())
@@ -176,4 +195,4 @@ def test_check_dependencies_succes(self, mock_log):
         mocked_devices = [mock_device, mock_device]
         Tests.check_dependencies(mocked_devices, mock_dependencies)
         assert mock_device.is_installed.call_count == 2
-        assert mock_log.call_count == 0
+        assert mock_log.call_count == 0