Skip to content

Commit 5dd9d5b

Browse files
billrestbillrest
committed
Bug 1562870 - generate conditioned profiles r=gbrown,bc,aerickson
Generates "conditioned" profiles. Differential Revision: https://phabricator.services.mozilla.com/D38211 --HG-- extra : moz-landing-system : lando
1 parent f39c078 commit 5dd9d5b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+8571
-0
lines changed

‎.flake8‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,13 @@ exclude =
2626
python/l10n/fluent_migrations,
2727
python/mozbuild/dumbmake,
2828
servo/components/style,
29+
testing/condprofile/condprof/android.py,
30+
testing/condprofile/condprof/creator.py,
31+
testing/condprofile/condprof/desktop.py,
32+
testing/condprofile/condprof/runner.py,
33+
testing/condprofile/condprof/scenarii/heavy.py,
34+
testing/condprofile/condprof/scenarii/cold.py,
35+
testing/condprofile/condprof/util.py,
2936
testing/jsshell/benchmark.py,
3037
testing/marionette/mach_commands.py,
3138
testing/mozharness/docs,

‎build/gen_test_packages_manifest.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
ALL_HARNESSES = [
1212
'common', # Harnesses without a specific package will look here.
13+
'condprof',
1314
'mochitest',
1415
'reftest',
1516
'xpcshell',
@@ -25,6 +26,7 @@
2526
]
2627

2728
PACKAGE_SPECIFIED_HARNESSES = [
29+
'condprof',
2830
'cppunittest',
2931
'mochitest',
3032
'reftest',

‎python/mozbuild/mozbuild/action/test_archive.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@
102102
'pattern': '**',
103103
'ignore': [
104104
'cppunittest/**',
105+
'condprof/**',
105106
'gtest/**',
106107
'mochitest/**',
107108
'reftest/**',
@@ -488,6 +489,37 @@
488489
'dest': 'talos/talos/tests/webkit/PerformanceTests/',
489490
},
490491
],
492+
'condprof': [
493+
{
494+
'source': buildconfig.topsrcdir,
495+
'base': 'testing',
496+
'pattern': 'condprofile/**',
497+
},
498+
{
499+
'source': buildconfig.topsrcdir,
500+
'base': 'testing/mozbase/mozfile',
501+
'pattern': '**',
502+
'dest': 'condprofile/mozfile',
503+
},
504+
{
505+
'source': buildconfig.topsrcdir,
506+
'base': 'testing/mozbase/mozprofile',
507+
'pattern': '**',
508+
'dest': 'condprofile/mozprofile',
509+
},
510+
{
511+
'source': buildconfig.topsrcdir,
512+
'base': 'testing/mozbase/mozdevice',
513+
'pattern': '**',
514+
'dest': 'condprofile/mozdevice',
515+
},
516+
{
517+
'source': buildconfig.topsrcdir,
518+
'base': 'third_party/python/virtualenv',
519+
'pattern': '**',
520+
'dest': 'condprofile/virtualenv',
521+
},
522+
],
491523
'raptor': [
492524
{
493525
'source': buildconfig.topsrcdir,

‎python/mozbuild/mozbuild/schedules.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
'windows',
4545
# test suites
4646
'awsy',
47+
'condprofile',
4748
'cppunittest',
4849
'firefox-ui',
4950
'geckoview',

‎taskcluster/ci/condprof/kind.yml‎

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
# This Source Code Form is subject to the terms of the Mozilla Public
2+
# License, v. 2.0. If a copy of the MPL was not distributed with this
3+
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4+
---
5+
loader: taskgraph.loader.transform:loader
6+
7+
transforms:
8+
- taskgraph.transforms.job:transforms
9+
- taskgraph.transforms.task:transforms
10+
11+
job-defaults:
12+
run-on-projects:
13+
- try
14+
- mozilla-central
15+
treeherder:
16+
kind: other
17+
tier: 2
18+
run:
19+
using: run-task
20+
checkout: false
21+
tooltool-downloads: public
22+
worker:
23+
max-run-time: 5400
24+
env:
25+
MOZ_FETCHES_DIR: fetches
26+
CONDPROF_ARCHIVES: /builds/worker/archive
27+
CONDPROF_ROOT: fetches/condprofile
28+
jobs:
29+
windows2012-64-firefox:
30+
worker-type: b-win2012
31+
description: Creates or updates conditioned profiles on Win64
32+
treeherder:
33+
symbol: condprof(firefox)
34+
platform: windows2012-64/opt
35+
index:
36+
product: firefox
37+
job-name: condprof-win64
38+
dependencies:
39+
build-win64/opt: build-win64/opt
40+
fetches:
41+
build-win64/opt:
42+
- artifact: target.common.tests.tar.gz
43+
- artifact: target.condprof.tests.tar.gz
44+
- artifact: target.zip
45+
run:
46+
using: run-task
47+
command: >-
48+
cd %CONDPROF_ROOT% &&
49+
python3.exe condprof/runner.py --geckodriver z:/%USERNAME%/fetches/bin/geckodriver.exe --firefox firefox/firefox.exe z:/%USERNAME%/archive
50+
worker:
51+
artifacts:
52+
- name: public/condprof
53+
path: archive
54+
type: directory
55+
linux64-firefox:
56+
worker-type: b-linux
57+
description: Creates or updates conditioned profiles on Linux
58+
treeherder:
59+
symbol: condprof(firefox)
60+
platform: linux64/opt
61+
index:
62+
product: firefox
63+
job-name: condprof-linux64
64+
dependencies:
65+
build-linux/opt: build-linux-shippable/opt
66+
fetches:
67+
build-linux/opt:
68+
- artifact: target.common.tests.tar.gz
69+
- artifact: target.condprof.tests.tar.gz
70+
- artifact: target.tar.bz2
71+
run:
72+
using: run-task
73+
command: >-
74+
$CONDPROF_ROOT/virtualenv/virtualenv.py -p python3.6 $CONDPROF_ROOT &&
75+
cd $CONDPROF_ROOT &&
76+
export PATH="$PATH:$MOZ_FETCHES_DIR/bin" &&
77+
bin/python3.6 condprof/runner.py --firefox $MOZ_FETCHES_DIR/firefox/firefox-bin $CONDPROF_ARCHIVES
78+
worker:
79+
docker-image: {in-tree: condprof}
80+
artifacts:
81+
- name: public/condprof
82+
path: /builds/worker/archive
83+
type: directory
84+
macosx64-firefox:
85+
worker-type: t-osx-1014
86+
description: Creates or updates conditioned profiles on macOS
87+
treeherder:
88+
symbol: condprof(firefox)
89+
platform: macosx64/opt
90+
dependencies:
91+
build-macosx64/opt: build-macosx64/opt
92+
fetches:
93+
build-macosx64/opt:
94+
- artifact: target.common.tests.tar.gz
95+
- artifact: target.condprof.tests.tar.gz
96+
- artifact: target.dmg
97+
index:
98+
product: firefox
99+
job-name: condprof-macosx64
100+
run:
101+
using: run-task
102+
command: >-
103+
$CONDPROF_ROOT/virtualenv/virtualenv.py -p python3 $CONDPROF_ROOT &&
104+
export PATH="$PATH:$MOZ_FETCHES_DIR/bin" &&
105+
cd $CONDPROF_ROOT &&
106+
bin/python3 condprof/runner.py --firefox $MOZ_FETCHES_DIR/target.dmg ../../archive
107+
worker:
108+
artifacts:
109+
- name: public/condprof
110+
path: archive
111+
type: directory
112+
android-hw-g5-7-0-arm7-api-16-geckoview:
113+
worker-type: t-bitbar-gw-perf-g5
114+
description: Creates or update conditioned profiles on GeckoView+G5
115+
treeherder:
116+
symbol: condprof(geckoview)
117+
platform: android-hw-g5-7-0-arm7-api-16/opt
118+
index:
119+
product: firefox
120+
job-name: condprof-g5-geckoview_example
121+
dependencies:
122+
build-linux/opt: build-linux-shippable/opt
123+
build-android-api-16/opt: build-android-api-16/opt
124+
fetches:
125+
build-linux/opt:
126+
- artifact: target.common.tests.tar.gz
127+
- artifact: target.condprof.tests.tar.gz
128+
build-android-api-16/opt:
129+
- artifact: geckoview_example.apk
130+
run:
131+
run-as-root: true
132+
using: run-task
133+
command: >-
134+
adb install -r $MOZ_FETCHES_DIR/geckoview_example.apk &&
135+
$CONDPROF_ROOT/virtualenv/virtualenv.py -p python3 $CONDPROF_ROOT &&
136+
cd $CONDPROF_ROOT &&
137+
export PATH="$PATH:$MOZ_FETCHES_DIR/bin" &&
138+
bin/python3 condprof/runner.py --device-name g5 --firefox org.mozilla.geckoview_example ../../archive
139+
worker:
140+
artifacts:
141+
- name: public/condprof
142+
path: archive
143+
type: directory
144+
android-hw-g5-7-0-arm7-api-16-fenix:
145+
worker-type: t-bitbar-gw-perf-g5
146+
description: Creates or update conditioned profiles on Fenix+G5
147+
treeherder:
148+
symbol: condprof(fenix)
149+
platform: android-hw-g5-7-0-arm7-api-16/opt
150+
index:
151+
product: firefox
152+
job-name: condprof-g5-fenix
153+
dependencies:
154+
build-linux/opt: build-linux-shippable/opt
155+
fetches:
156+
build-linux/opt:
157+
- artifact: target.common.tests.tar.gz
158+
- artifact: target.condprof.tests.tar.gz
159+
run:
160+
run-as-root: true
161+
command: >-
162+
curl -L -o target.apk https://index.taskcluster.net/v1/task/project.mobile.fenix.v2.nightly.latest/artifacts/public/build/armeabi-v7a/geckoNightly/target.apk &&
163+
adb install -r target.apk &&
164+
rm -rf target.apk &&
165+
$CONDPROF_ROOT/virtualenv/virtualenv.py -p python3 $CONDPROF_ROOT &&
166+
cd $CONDPROF_ROOT &&
167+
export PATH="$PATH:$MOZ_FETCHES_DIR/bin" &&
168+
bin/python3 condprof/runner.py --device-name g5 --firefox org.mozilla.fenix.nightly ../../archive
169+
worker:
170+
artifacts:
171+
- name: public/condprof
172+
path: archive
173+
type: directory
174+
android-hw-p2-8-0-aarch64-fenix:
175+
worker-type: t-bitbar-gw-perf-p2
176+
description: Creates or update conditioned profiles on Fenix+P2
177+
treeherder:
178+
symbol: condprof(fenix)
179+
platform: android-hw-p2-8-0-android-aarch64/opt
180+
dependencies:
181+
build-linux/opt: build-linux-shippable/opt
182+
fetches:
183+
build-linux/opt:
184+
- artifact: target.common.tests.tar.gz
185+
- artifact: target.condprof.tests.tar.gz
186+
index:
187+
product: firefox
188+
job-name: condprof-p2_aarch64-fenix
189+
run:
190+
run-as-root: true
191+
command: >-
192+
curl -L -o target.apk https://index.taskcluster.net/v1/task/project.mobile.fenix.v2.nightly.latest/artifacts/public/build/arm64-v8a/geckoNightly/target.apk &&
193+
adb install -r target.apk &&
194+
rm -rf target.apk &&
195+
$CONDPROF_ROOT/virtualenv/virtualenv.py -p python3 $CONDPROF_ROOT &&
196+
cd $CONDPROF_ROOT &&
197+
export PATH="$PATH:$MOZ_FETCHES_DIR/bin" &&
198+
bin/python3 condprof/runner.py --device-name p2_aarch64 --firefox org.mozilla.fenix.nightly ../../archive
199+
worker:
200+
artifacts:
201+
- name: public/condprof
202+
path: archive
203+
type: directory
204+
android-hw-p2-8-0-android-aarch64-geckoview:
205+
worker-type: t-bitbar-gw-perf-p2
206+
description: Creates or update conditioned profiles on GeckoView+P2
207+
treeherder:
208+
symbol: condprof(geckoview)
209+
platform: android-hw-p2-8-0-android-aarch64/opt
210+
dependencies:
211+
build-linux/opt: build-linux-shippable/opt
212+
build-android-api-16/opt: build-android-api-16/opt
213+
index:
214+
product: firefox
215+
job-name: condprof-p2_aarch64-geckoview_example
216+
fetches:
217+
build-linux/opt:
218+
- artifact: target.common.tests.tar.gz
219+
- artifact: target.condprof.tests.tar.gz
220+
build-android-api-16/opt:
221+
- artifact: geckoview_example.apk
222+
run:
223+
run-as-root: true
224+
using: run-task
225+
command: >-
226+
adb install -r $MOZ_FETCHES_DIR/geckoview_example.apk &&
227+
$CONDPROF_ROOT/virtualenv/virtualenv.py -p python3 $CONDPROF_ROOT &&
228+
cd $CONDPROF_ROOT &&
229+
export PATH="$PATH:$MOZ_FETCHES_DIR/bin" &&
230+
bin/python3 condprof/runner.py --device-name p2_aarch64 --firefox org.mozilla.geckoview_example ../../archive
231+
worker:
232+
artifacts:
233+
- name: public/condprof
234+
path: archive
235+
type: directory

‎taskcluster/ci/config.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ treeherder:
138138
'SS': 'Shadow scheduler'
139139
'test-info': 'Test manifest skip/fail information'
140140
'vismet': 'Visual Metrics Analsyis'
141+
'condprof': 'Conditioned Profile Builder'
141142

142143
index:
143144
products:

‎taskcluster/ci/docker-image/kind.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,9 @@ jobs:
230230
webrender:
231231
symbol: I(webrender)
232232
parent: debian9-base
233+
condprof:
234+
symbol: I(condprof)
235+
parent: desktop1604-test
233236
webrender-updater:
234237
symbol: I(wrupdater)
235238
parent: debian9-base

‎taskcluster/docker/condprof/Dockerfile‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# %ARG DOCKER_IMAGE_PARENT
2+
FROM $DOCKER_IMAGE_PARENT
3+
MAINTAINER Tarek Ziade <tarek@mozilla.com>
4+
5+
VOLUME /builds/worker/.cache
6+
VOLUME /builds/worker/checkouts
7+
VOLUME /builds/worker/tooltool-cache
8+
VOLUME /builds/worker/workspace
9+
10+
RUN add-apt-repository ppa:jonathonf/python-3.6
11+
RUN apt-get -y update && \
12+
apt-get install -y python3.6 && \
13+
curl https://bootstrap.pypa.io/get-pip.py | python3.6
14+
15+
CMD ["/bin/bash", "--login"]
16+

‎taskcluster/docs/kinds.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,10 @@ beetmover-geckoview
224224
-------------------
225225
Beetmover-geckoview publishes the Android library called "geckoview".
226226

227+
condprof
228+
--------
229+
condprof creates and updates realistic profiles.
230+
227231
release-source-checksums-signing
228232
--------------------------------
229233
release-source-checksums-signing take as input the checksums file generated by

‎testing/condprofile/Makefile‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
HERE = $(shell pwd)
2+
BIN = $(HERE)/bin
3+
PYTHON = $(BIN)/python
4+
INSTALL = $(BIN)/pip install --no-deps
5+
BUILD_DIRS = bin build include lib lib64 man share
6+
VIRTUALENV = virtualenv
7+
8+
.PHONY: all test build clean docs
9+
10+
all: build
11+
12+
$(PYTHON):
13+
$(VIRTUALENV) $(VTENV_OPTS) .
14+
15+
build: $(PYTHON)
16+
$(PYTHON) setup.py develop
17+
$(BIN)/pip install tox
18+
19+
clean:
20+
rm -rf $(BUILD_DIRS)
21+
22+
test: build
23+
$(BIN)/tox
24+
25+
docs: build
26+
$(BIN)/tox -e docs

0 commit comments

Comments
 (0)