report: Use taskcluster client library, fixes #386 (#390)

Author: La0

.taskcluster.yml

@@ -149,7 +149,7 @@ tasks:
           - sh
           - -lxce
           - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks &&
-            cd /src/report && ./ci-test.sh"
+            cd /src/report && pip install -r requirements.txt && ./ci-test.sh"
       metadata:
         name: "Code Coverage Report checks: unit tests"
         description: Check python code with unittest

report/firefox_code_coverage/__init__.py

@@ -1 +1,8 @@
 # -*- coding: utf-8 -*-
+
+from taskcluster.helper import TaskclusterConfig
+
+taskcluster = TaskclusterConfig("https://firefox-ci-tc.services.mozilla.com")
+
+# Force root url to avoid proxy as grcov is not available on Community instance
+taskcluster.options = {"rootUrl": taskcluster.default_url}

report/firefox_code_coverage/codecoverage.py

@@ -12,13 +12,9 @@
 import time
 import warnings
 
-try:
-    from urllib.parse import urlencode
-    from urllib.request import Request, urlopen, urlretrieve
-except ImportError:
-    from urllib import urlencode, urlretrieve
-    from urllib2 import Request, urlopen
+import requests
 
+from firefox_code_coverage import taskcluster
 
 FINISHED_STATUSES = ["completed", "failed", "exception"]
 ALL_STATUSES = FINISHED_STATUSES + ["unscheduled", "pending", "running"]
@@ -28,91 +24,87 @@
 GRCOV_ARTIFACT = "public/build/grcov.tar.xz"
 
 
-def get_json(url, params=None, headers={}):
-    if params is not None:
-        url += "?" + urlencode(params)
-
-    request = Request(url, headers=headers)
-    r = urlopen(request).read().decode("utf-8")
-
-    return json.loads(r)
-
-
 def is_taskcluster_loaner():
     return "TASKCLUSTER_INTERACTIVE" in os.environ
 
 
 def get_task(branch, revision):
-    task = get_json(
-        f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.{branch}.revision.{revision}.firefox.decision"
-    )
+    index = taskcluster.get_service("index")
+    task = index.findTask(f"gecko.v2.{branch}.revision.{revision}.firefox.decision")
     return task["taskId"]
 
 
 def get_last_task():
-    revision = get_json(
+    resp = requests.get(
         "https://api.coverage.moz.tools/v2/latest?repository=mozilla-central"
-    )[0]["revision"]
+    )
+    resp.raise_for_status()
+    data = resp.json()
+    revision = data[0]["revision"]
     return get_task("mozilla-central", revision)
 
 
 def get_task_details(task_id):
-    task_details = get_json(
-        "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/" + task_id
-    )
-    return task_details
+    queue = taskcluster.get_service("queue")
+    return queue.task(task_id)
 
 
 def get_task_artifacts(task_id):
-    artifacts = get_json(
-        "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/"
-        + task_id
-        + "/artifacts"
-    )
-    return artifacts["artifacts"]
+    queue = taskcluster.get_service("queue")
+    response = queue.listLatestArtifacts(task_id)
+    return response["artifacts"]
 
 
 def get_tasks_in_group(group_id):
-    reply = get_json(
-        "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task-group/"
-        + group_id
-        + "/list",
-        {"limit": "200"},
-    )
-    tasks = reply["tasks"]
-    while "continuationToken" in reply:
-        reply = get_json(
-            "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task-group/"
-            + group_id
-            + "/list",
-            {"limit": "200", "continuationToken": reply["continuationToken"]},
-        )
-        tasks += reply["tasks"]
+    tasks = []
+
+    def _save_tasks(response):
+        tasks.extend(response["tasks"])
+
+    queue = taskcluster.get_service("queue")
+    queue.listTaskGroup(group_id, paginationHandler=_save_tasks)
+
     return tasks
 
 
+def download_binary(url, path, retries=5):
+    """Download a binary file from an url"""
+    for i in range(1, retries + 1):
+        try:
+            artifact = requests.get(url, stream=True)
+            artifact.raise_for_status()
+
+            with open(path, "wb") as f:
+                for chunk in artifact.iter_content(chunk_size=8192):
+                    f.write(chunk)
+            break
+        except:  # noqa: E722
+            try:
+                os.remove(path)
+            except OSError:
+                pass
+
+            if i == retries:
+                raise Exception(
+                    "Download failed after {} retries - {}".format(retries, url)
+                )
+
+            time.sleep(7 * i)
+
+
 def download_artifact(task_id, artifact, artifacts_path):
     fname = os.path.join(
         artifacts_path, task_id + "_" + os.path.basename(artifact["name"])
     )
+
+    # As recommended by Taskcluster doc, use requests to download
+    # from the artifact public url instead of relying on the client method
+    queue = taskcluster.get_service("queue")
+    url = queue.buildUrl("getLatestArtifact", task_id, artifact["name"])
+
     if not os.path.exists(fname):
-        while True:
-            try:
-                urlretrieve(
-                    "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/"
-                    + task_id
-                    + "/artifacts/"
-                    + artifact["name"],
-                    fname,
-                )
-                break
-            except:  # noqa: E722
-                try:
-                    os.remove(fname)
-                except OSError:
-                    pass
-
-                time.sleep(7)
+        download_binary(url, fname)
+
     return fname
 
 
@@ -145,11 +137,8 @@ def get_platform(task_name):
 
 
 def get_task_status(task_id):
-    status = get_json(
-        "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{}/status".format(
-            task_id
-        )
-    )
+    queue = taskcluster.get_service("queue")
+    status = queue.status(task_id)
     return status["status"]["state"]
 
 
@@ -316,10 +305,9 @@ def download_grcov():
 
     dest = tempfile.mkdtemp(suffix="grcov")
     archive = os.path.join(dest, "grcov.tar.xz")
-    url = "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/{index}/artifacts/{artifact}".format(
-        index=GRCOV_INDEX, artifact=GRCOV_ARTIFACT
-    )
-    urlretrieve(url, archive)
+    index = taskcluster.get_service("index")
+    url = index.buildUrl("findArtifactFromTask", GRCOV_INDEX, GRCOV_ARTIFACT)
+    download_binary(url, archive)
 
     # Extract archive in temp
     tar = tarfile.open(archive, "r:xz")

report/requirements.txt

@@ -0,0 +1 @@
+taskcluster==24.2.0