-
Notifications
You must be signed in to change notification settings - Fork 54
/
mach_commands.py
273 lines (228 loc) · 10.3 KB
/
mach_commands.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Originally taken from /talos/mach_commands.py
# Integrates raptor mozharness with mach
from __future__ import absolute_import, print_function, unicode_literals
import json
import logging
import os
import shutil
import socket
import subprocess
import sys
import mozfile
from mach.decorators import Command, CommandProvider
from mozboot.util import get_state_dir
from mozbuild.base import (
MachCommandBase,
MozbuildObject,
BinaryNotFoundException,
)
from mozbuild.base import MachCommandConditions as Conditions
HERE = os.path.dirname(os.path.realpath(__file__))
BENCHMARK_REPOSITORY = 'https://github.com/mozilla/perf-automation'
BENCHMARK_REVISION = 'e19a0865c946ae2f9a64dd25614b1c275a3996b2'
ANDROID_BROWSERS = ["fennec", "geckoview", "refbrow", "fenix", "chrome-m"]
class RaptorRunner(MozbuildObject):
def run_test(self, raptor_args, kwargs):
"""Setup and run mozharness.
We want to do a few things before running Raptor:
1. Clone mozharness
2. Make the config for Raptor mozharness
3. Run mozharness
"""
self.init_variables(raptor_args, kwargs)
self.setup_benchmarks()
self.make_config()
self.write_config()
self.make_args()
return self.run_mozharness()
def init_variables(self, raptor_args, kwargs):
self.raptor_args = raptor_args
if kwargs.get('host') == 'HOST_IP':
kwargs['host'] = os.environ['HOST_IP']
self.host = kwargs['host']
self.is_release_build = kwargs['is_release_build']
self.memory_test = kwargs['memory_test']
self.power_test = kwargs['power_test']
self.cpu_test = kwargs['cpu_test']
self.live_sites = kwargs['live_sites']
self.disable_perf_tuning = kwargs['disable_perf_tuning']
self.conditioned_profile_scenario = kwargs['conditioned_profile_scenario']
self.device_name = kwargs['device_name']
if Conditions.is_android(self) or kwargs["app"] in ANDROID_BROWSERS:
self.binary_path = None
else:
self.binary_path = kwargs.get("binary") or self.get_binary_path()
self.python = sys.executable
self.raptor_dir = os.path.join(self.topsrcdir, 'testing', 'raptor')
self.mozharness_dir = os.path.join(self.topsrcdir, 'testing', 'mozharness')
self.config_file_path = os.path.join(
self._topobjdir, 'testing', 'raptor-in_tree_conf.json')
self.virtualenv_script = os.path.join(
self.topsrcdir, 'third_party', 'python', 'virtualenv', 'virtualenv.py')
self.virtualenv_path = os.path.join(
self._topobjdir, 'testing', 'raptor-venv')
def setup_benchmarks(self):
"""Make sure benchmarks are linked to the proper location in the objdir.
Benchmarks can either live in-tree or in an external repository. In the latter
case also clone/update the repository if necessary.
"""
external_repo_path = os.path.join(get_state_dir(), 'performance-tests')
print("Updating external benchmarks from {}".format(BENCHMARK_REPOSITORY))
print("Cloning the benchmarks to {}".format(external_repo_path))
try:
subprocess.check_output(['git', '--version'])
except Exception as ex:
print("Git is not available! Please install git and "
"ensure it is included in the terminal path")
raise ex
if not os.path.isdir(external_repo_path):
subprocess.check_call(['git', 'clone', BENCHMARK_REPOSITORY, external_repo_path])
else:
subprocess.check_call(['git', 'checkout', 'master'], cwd=external_repo_path)
subprocess.check_call(['git', 'pull'], cwd=external_repo_path)
subprocess.check_call(['git', 'checkout', BENCHMARK_REVISION], cwd=external_repo_path)
# Link or copy benchmarks to the objdir
benchmark_paths = (
os.path.join(external_repo_path, 'benchmarks'),
os.path.join(self.topsrcdir, 'third_party', 'webkit', 'PerformanceTests'),
)
benchmark_dest = os.path.join(self.topobjdir, 'testing', 'raptor', 'benchmarks')
if not os.path.isdir(benchmark_dest):
os.makedirs(benchmark_dest)
for benchmark_path in benchmark_paths:
for name in os.listdir(benchmark_path):
path = os.path.join(benchmark_path, name)
dest = os.path.join(benchmark_dest, name)
if not os.path.isdir(path) or name.startswith('.'):
continue
if hasattr(os, 'symlink'):
if not os.path.exists(dest):
os.symlink(path, dest)
else:
# Clobber the benchmark in case a recent update removed any files.
mozfile.remove(dest)
shutil.copytree(path, dest)
def make_config(self):
default_actions = [
'populate-webroot',
'install-chromium-distribution',
'create-virtualenv',
'run-tests'
]
self.config = {
'run_local': True,
'binary_path': self.binary_path,
'repo_path': self.topsrcdir,
'raptor_path': self.raptor_dir,
'obj_path': self.topobjdir,
'log_name': 'raptor',
'virtualenv_path': self.virtualenv_path,
'pypi_url': 'http://pypi.org/simple',
'base_work_dir': self.mozharness_dir,
'exes': {
'python': self.python,
'virtualenv': [self.python, self.virtualenv_script],
},
'title': socket.gethostname(),
'default_actions': default_actions,
'raptor_cmd_line_args': self.raptor_args,
'host': self.host,
'power_test': self.power_test,
'memory_test': self.memory_test,
'cpu_test': self.cpu_test,
'live_sites': self.live_sites,
'disable_perf_tuning': self.disable_perf_tuning,
'conditioned_profile_scenario': self.conditioned_profile_scenario,
'is_release_build': self.is_release_build,
'device_name': self.device_name,
}
sys.path.insert(0, os.path.join(self.topsrcdir, 'tools', 'browsertime'))
try:
import mach_commands as browsertime
# We don't set `browsertime_{chromedriver,geckodriver} -- those will be found by
# browsertime in its `node_modules` directory, which is appropriate for local builds.
# We don't set `browsertime_ffmpeg` yet: it will need to be on the path. There is code
# to configure the environment including the path in
# `tools/browsertime/mach_commands.py` but integrating it here will take more effort.
self.config.update({
'browsertime_node': browsertime.node_path(),
'browsertime_browsertimejs': browsertime.browsertime_path(),
})
finally:
sys.path = sys.path[1:]
def make_args(self):
self.args = {
'config': {},
'initial_config_file': self.config_file_path,
}
def write_config(self):
try:
config_file = open(self.config_file_path, 'w')
config_file.write(json.dumps(self.config))
config_file.close()
except IOError as e:
err_str = "Error writing to Raptor Mozharness config file {0}:{1}"
print(err_str.format(self.config_file_path, str(e)))
raise e
def run_mozharness(self):
sys.path.insert(0, self.mozharness_dir)
from mozharness.mozilla.testing.raptor import Raptor
raptor_mh = Raptor(config=self.args['config'],
initial_config_file=self.args['initial_config_file'])
return raptor_mh.run()
def create_parser():
sys.path.insert(0, HERE) # allow to import the raptor package
from raptor.cmdline import create_parser
return create_parser(mach_interface=True)
@CommandProvider
class MachRaptor(MachCommandBase):
@Command('raptor', category='testing',
description='Run Raptor performance tests.',
parser=create_parser)
def run_raptor(self, **kwargs):
# Defers this import so that a transitive dependency doesn't
# stop |mach bootstrap| from running
from raptor.power import enable_charging, disable_charging
build_obj = self
is_android = Conditions.is_android(build_obj) or \
kwargs['app'] in ANDROID_BROWSERS
if is_android:
from mozrunner.devices.android_device import (verify_android_device, InstallIntent)
from mozdevice import ADBAndroid
install = InstallIntent.NO if kwargs.pop('noinstall', False) else InstallIntent.YES
if not verify_android_device(build_obj, install=install,
app=kwargs['binary'],
xre=True): # Equivalent to 'run_local' = True.
return 1
debug_command = '--debug-command'
if debug_command in sys.argv:
sys.argv.remove(debug_command)
raptor = self._spawn(RaptorRunner)
device = None
try:
if kwargs['power_test'] and is_android:
device = ADBAndroid(verbose=True)
disable_charging(device)
return raptor.run_test(sys.argv[2:], kwargs)
except BinaryNotFoundException as e:
self.log(logging.ERROR, 'raptor',
{'error': str(e)},
'ERROR: {error}')
self.log(logging.INFO, 'raptor',
{'help': e.help()},
'{help}')
return 1
except Exception as e:
print(repr(e))
return 1
finally:
if kwargs['power_test'] and device:
enable_charging(device)
@Command('raptor-test', category='testing',
description='Run Raptor performance tests.',
parser=create_parser)
def run_raptor_test(self, **kwargs):
return self.run_raptor(**kwargs)