Skip to content

Commit 020fc57

Browse files
marco-cmarco-c
authored
Stop relying on task routes specific to platform and configuration (#375)
1 parent 7394744 commit 020fc57

File tree

10 files changed

+171
-112
lines changed

10 files changed

+171
-112
lines changed

‎bot/code_coverage_bot/artifacts.py‎

Lines changed: 6 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@
2626

2727

2828
class ArtifactsHandler(object):
29-
def __init__(self, task_ids, parent_dir="ccov-artifacts", task_name_filter="*"):
30-
self.task_ids = task_ids
29+
def __init__(self, test_tasks, parent_dir="ccov-artifacts", task_name_filter="*"):
30+
self.test_tasks = test_tasks
3131
self.parent_dir = parent_dir
3232
self.task_name_filter = task_name_filter
3333
self.artifacts = []
@@ -133,26 +133,9 @@ def is_filtered_task(self, task):
133133
def download_all(self):
134134
os.makedirs(self.parent_dir, exist_ok=True)
135135

136-
# The test tasks for the Linux and Windows builds are in the same group,
137-
# but the following code is generic and supports build tasks split in
138-
# separate groups.
139-
groups = set(
140-
[
141-
taskcluster.get_task_details(build_task_id)["taskGroupId"]
142-
for build_task_id in self.task_ids.values()
143-
if build_task_id is not None
144-
]
145-
)
146-
test_tasks = [
147-
task
148-
for group in groups
149-
for task in taskcluster.get_tasks_in_group(group)
150-
if taskcluster.is_coverage_task(task["task"])
151-
and not self.is_filtered_task(task)
152-
]
153-
logger.info("Downloading artifacts from {} tasks".format(len(test_tasks)))
154-
155-
for test_task in test_tasks:
136+
logger.info("Downloading artifacts from {} tasks".format(len(self.test_tasks)))
137+
138+
for test_task in self.test_tasks:
156139
status = test_task["status"]["state"]
157140
task_id = test_task["status"]["taskId"]
158141
while status not in FINISHED_STATUSES:
@@ -168,7 +151,7 @@ def download_all(self):
168151

169152
# Choose best tasks to download (e.g. 'completed' is better than 'failed')
170153
download_tasks = {}
171-
for test_task in test_tasks:
154+
for test_task in self.test_tasks:
172155
status = test_task["status"]["state"]
173156
assert status in FINISHED_STATUSES, "State '{}' not recognized".format(
174157
status

‎bot/code_coverage_bot/hooks/base.py‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,6 @@
2020
logger = structlog.get_logger(__name__)
2121

2222

23-
PLATFORMS = ["linux", "windows", "android-test", "android-emulator"]
24-
25-
2623
class Hook(object):
2724
def __init__(
2825
self,
@@ -54,21 +51,26 @@ def __init__(
5451
assert os.path.isdir(cache_root), f"Cache root {cache_root} is not a dir."
5552
self.repo_dir = os.path.join(cache_root, self.branch)
5653

57-
# Load current coverage task for all platforms
58-
task_ids = {
59-
platform: taskcluster.get_task(self.branch, self.revision, platform)
60-
for platform in PLATFORMS
61-
}
54+
# Load coverage tasks for all platforms
55+
decision_task_id = taskcluster.get_decision_task(self.branch, self.revision)
56+
57+
group = taskcluster.get_task_details(decision_task_id)["taskGroupId"]
58+
59+
test_tasks = [
60+
task
61+
for task in taskcluster.get_tasks_in_group(group)
62+
if taskcluster.is_coverage_task(task["task"])
63+
]
6264

6365
# Check the required platforms are present
66+
platforms = set(
67+
taskcluster.get_platform(test_task["task"]) for test_task in test_tasks
68+
)
6469
for platform in required_platforms:
65-
if not task_ids[platform]:
66-
raise Exception(
67-
f"Code coverage build on {platform} failed and was not indexed."
68-
)
70+
assert platform in platforms, f"{platform} missing in the task group."
6971

7072
self.artifactsHandler = ArtifactsHandler(
71-
task_ids, self.artifacts_dir, task_name_filter
73+
test_tasks, self.artifacts_dir, task_name_filter
7274
)
7375

7476
@property

‎bot/code_coverage_bot/taskcluster.py‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,23 +17,8 @@
1717
NAME_PARTS_TO_SKIP = ("opt", "debug", "e10s", "1proc")
1818

1919

20-
def get_task(branch, revision, platform):
21-
if platform == "linux":
22-
platform_name = "linux64-ccov-opt"
23-
product = "firefox"
24-
elif platform == "windows":
25-
platform_name = "win64-ccov-opt"
26-
product = "firefox"
27-
elif platform == "android-test":
28-
platform_name = "android-test-ccov"
29-
product = "mobile"
30-
elif platform == "android-emulator":
31-
platform_name = "android-api-16-ccov-debug"
32-
product = "mobile"
33-
else:
34-
raise Exception(f"Unsupported platform: {platform}")
35-
36-
route = f"gecko.v2.{branch}.revision.{revision}.{product}.{platform_name}"
20+
def get_decision_task(branch, revision):
21+
route = f"gecko.v2.{branch}.revision.{revision}.firefox.decision"
3722
index = taskcluster_config.get_service("index")
3823
try:
3924
return index.findTask(route)["taskId"]

‎bot/tests/conftest.py‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,18 @@ def TASK_NOT_FOUND():
6868

6969

7070
@pytest.fixture(scope="session")
71-
def LATEST_LINUX():
72-
return load_json("latest_linux.json")
71+
def DECISION_TASK_ID():
72+
return "OuvSoOjkSvKYLbaGMknMfA"
73+
74+
75+
@pytest.fixture(scope="session")
76+
def DECISION_TASK():
77+
return load_json("decision_task.json")
78+
79+
80+
@pytest.fixture(scope="session")
81+
def LATEST_DECISION():
82+
return load_json("latest_decision.json")
7383

7484

7585
@pytest.fixture(scope="session")
@@ -92,16 +102,6 @@ def LINUX_TASK_ARTIFACTS():
92102
return load_json("linux_task_artifacts.json")
93103

94104

95-
@pytest.fixture(scope="session")
96-
def LATEST_WIN():
97-
return load_json("latest_win.json")
98-
99-
100-
@pytest.fixture(scope="session")
101-
def WIN_TASK_ID():
102-
return "PWnw3h-QQSiqxO83MDzKag"
103-
104-
105105
@pytest.fixture(scope="session")
106106
def GROUP_TASKS_1():
107107
return load_json("task-group_1.json")

‎bot/tests/fixtures/decision_task.json‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
{
2+
"provisionerId": "gecko-3",
3+
"workerType": "decision",
4+
"schedulerId": "gecko-level-3",
5+
"taskGroupId": "OuvSoOjkSvKYLbaGMknMfA",
6+
"dependencies": [],
7+
"requires": "all-completed",
8+
"routes": [
9+
"tc-treeherder.v2.mozilla-central.7828a10a94b6afb78d18d9b7b83e7aa79337cc24.37010",
10+
"index.gecko.v2.mozilla-central.latest.taskgraph.decision",
11+
"index.gecko.v2.mozilla-central.revision.7828a10a94b6afb78d18d9b7b83e7aa79337cc24.taskgraph.decision",
12+
"index.gecko.v2.mozilla-central.pushlog-id.37010.decision",
13+
"notify.email.dvarga@mozilla.com.on-failed",
14+
"notify.email.dvarga@mozilla.com.on-exception",
15+
"index.gecko.v2.mozilla-central.latest.firefox.decision",
16+
"index.gecko.v2.mozilla-central.revision.7828a10a94b6afb78d18d9b7b83e7aa79337cc24.firefox.decision"
17+
],
18+
"priority": "lowest",
19+
"retries": 5,
20+
"created": "2020-01-13T15:45:58.646Z",
21+
"deadline": "2020-01-14T15:45:58.646Z",
22+
"expires": "2021-01-12T15:45:58.646Z",
23+
"scopes": [
24+
"assume:repo:hg.mozilla.org/mozilla-central:branch:default",
25+
"queue:route:notify.email.dvarga@mozilla.com.*",
26+
"in-tree:hook-action:project-gecko/in-tree-action-3-*"
27+
],
28+
"payload": {
29+
"env": {
30+
"GECKO_BASE_REPOSITORY": "https://hg.mozilla.org/mozilla-unified",
31+
"GECKO_HEAD_REPOSITORY": "https://hg.mozilla.org/mozilla-central",
32+
"GECKO_HEAD_REF": "7828a10a94b6afb78d18d9b7b83e7aa79337cc24",
33+
"GECKO_HEAD_REV": "7828a10a94b6afb78d18d9b7b83e7aa79337cc24",
34+
"HG_STORE_PATH": "/builds/worker/checkouts/hg-store",
35+
"TASKCLUSTER_CACHES": "/builds/worker/checkouts",
36+
"PYTHONDONTWRITEBYTECODE": "1",
37+
"TASKCLUSTER_ROOT_URL": "https://taskcluster.net",
38+
"TASKCLUSTER_PROXY_URL": "http://taskcluster"
39+
},
40+
"cache": {
41+
"gecko-level-3-checkouts-sparse-v2": "/builds/worker/checkouts"
42+
},
43+
"features": {
44+
"taskclusterProxy": true,
45+
"chainOfTrust": true
46+
},
47+
"image": "taskcluster/decision:2.2.0@sha256:0e9689e94605eb8395f5b49141a48148416b0d825f6f7be04c29642d1a85ee3d",
48+
"maxRunTime": 1800,
49+
"command": [
50+
"/builds/worker/bin/run-task",
51+
"--gecko-checkout=/builds/worker/checkouts/gecko",
52+
"--gecko-sparse-profile=build/sparse-profiles/taskgraph",
53+
"--",
54+
"bash",
55+
"-cx",
56+
"cd /builds/worker/checkouts/gecko && ln -s /builds/worker/artifacts artifacts && ./mach --log-no-times taskgraph decision --pushlog-id='37010' --pushdate='1578930324' --project='mozilla-central' --owner='dvarga@mozilla.com' --level='3' --tasks-for='hg-push' --base-repository=\"$GECKO_BASE_REPOSITORY\" --head-repository=\"$GECKO_HEAD_REPOSITORY\" --head-ref=\"$GECKO_HEAD_REF\" --head-rev=\"$GECKO_HEAD_REV\" \n"
57+
],
58+
"artifacts": {
59+
"public": {
60+
"type": "directory",
61+
"path": "/builds/worker/artifacts",
62+
"expires": "2021-01-12T15:45:58.646Z"
63+
}
64+
}
65+
},
66+
"metadata": {
67+
"owner": "dvarga@mozilla.com",
68+
"source": "https://hg.mozilla.org/mozilla-central/raw-file/7828a10a94b6afb78d18d9b7b83e7aa79337cc24/.taskcluster.yml",
69+
"name": "Gecko Decision Task",
70+
"description": "The task that creates all of the other tasks in the task graph"
71+
},
72+
"tags": {
73+
"createdForUser": "dvarga@mozilla.com",
74+
"kind": "decision-task"
75+
},
76+
"extra": {
77+
"treeherder": {
78+
"machine": {
79+
"platform": "gecko-decision"
80+
},
81+
"symbol": "D"
82+
},
83+
"tasks_for": "hg-push",
84+
"notify": {
85+
"email": {
86+
"link": {
87+
"text": "Treeherder Jobs",
88+
"href": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central&revision=7828a10a94b6afb78d18d9b7b83e7aa79337cc24"
89+
}
90+
}
91+
}
92+
}
93+
}

‎bot/tests/fixtures/latest_decision.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"namespace": "gecko.v2.mozilla-central.revision.7828a10a94b6afb78d18d9b7b83e7aa79337cc24.firefox.decision",
3+
"taskId": "OuvSoOjkSvKYLbaGMknMfA",
4+
"rank": 0,
5+
"data": {},
6+
"expires": "2021-01-12T15:45:58.646Z"
7+
}

‎bot/tests/fixtures/latest_linux.json‎

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎bot/tests/fixtures/latest_win.json‎

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎bot/tests/test_artifacts.py‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
from code_coverage_bot.artifacts import Artifact
1111
from code_coverage_bot.artifacts import ArtifactsHandler
12+
from code_coverage_bot.hooks.base import Hook
1213

1314
FILES = [
1415
"windows_mochitest-1_code-coverage-jsvm.info",
@@ -259,28 +260,45 @@ def build_task(task_state):
259260

260261

261262
def test_download_all(
262-
LINUX_TASK_ID,
263-
LINUX_TASK,
263+
DECISION_TASK_ID,
264+
DECISION_TASK,
265+
LATEST_DECISION,
264266
GROUP_TASKS_1,
265267
GROUP_TASKS_2,
266268
fake_artifacts,
267269
mock_taskcluster,
270+
tmpdir,
268271
):
269272
responses.add(
270273
responses.GET,
271-
f"http://taskcluster.test/api/queue/v1/task/{LINUX_TASK_ID}",
272-
json=LINUX_TASK,
274+
"http://taskcluster.test/api/index/v1/task/gecko.v2.mozilla-central.revision.7828a10a94b6afb78d18d9b7b83e7aa79337cc24.firefox.decision",
275+
json=LATEST_DECISION,
276+
status=200,
277+
)
278+
responses.add(
279+
responses.GET,
280+
f"http://taskcluster.test/api/queue/v1/task/{DECISION_TASK_ID}",
281+
json=DECISION_TASK,
273282
status=200,
274283
)
275284
for group_tasks in _group_tasks():
276285
responses.add(
277286
responses.GET,
278-
"http://taskcluster.test/api/queue/v1/task-group/aPt9FbIdQwmhwDIPDYLuaw/list",
287+
"http://taskcluster.test/api/queue/v1/task-group/OuvSoOjkSvKYLbaGMknMfA/list",
279288
json=group_tasks,
280289
status=200,
281290
)
282291

283-
a = ArtifactsHandler({"linux": LINUX_TASK_ID})
292+
h = Hook(
293+
"https://hg.mozilla.org/mozilla-central",
294+
"7828a10a94b6afb78d18d9b7b83e7aa79337cc24",
295+
"*",
296+
tmpdir,
297+
tmpdir,
298+
)
299+
a = h.artifactsHandler
300+
301+
# a = ArtifactsHandler({"linux": LINUX_TASK_ID})
284302

285303
downloaded = set()
286304

0 commit comments

Comments
 (0)