diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 87e01acf0307..67e8329bb84f 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -4,21 +4,18 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -import hashlib -import json import os.path -import subprocess -from decisionlib import DecisionTask +from decisionlib import * -def main(): - task_for = os.environ["TASK_FOR"] - +def main(task_for, mock=False): if task_for == "github-push": - linux_tidy_unit() - #linux_wpt() - android_arm32() - windows_dev() + if CONFIG.git_ref in ["refs/heads/auto", "refs/heads/try", "refs/heads/try-taskcluster"]: + linux_tidy_unit() + android_arm32() + windows_dev() + if mock: + linux_wpt() # https://tools.taskcluster.net/hooks/project-servo/daily elif task_for == "daily": @@ -31,206 +28,89 @@ def main(): ping_on_daily_task_failure = "SimonSapin, nox, emilio" -build_artifacts_expiry = "1 week" -build_dependencies_artifacts_expiry = "1 month" -log_artifacts_expiry = "1 year" +build_artifacts_expire_in = "1 week" +build_dependencies_artifacts_expire_in = "1 month" +log_artifacts_expire_in = "1 year" build_env = { "RUST_BACKTRACE": "1", "RUSTFLAGS": "-Dwarnings", "CARGO_INCREMENTAL": "0", - "SCCACHE_IDLE_TIMEOUT": "1200", } -linux_build_env = dict(**build_env, **{ +linux_build_env = { "CCACHE": "sccache", "RUSTC_WRAPPER": "sccache", + "SCCACHE_IDLE_TIMEOUT": "1200", "SHELL": "/bin/dash", # For SpiderMonkey’s build system -}) +} +windows_build_env = {} +windows_sparse_checkout = [ + "/*", + "!/tests/wpt/metadata", + "!/tests/wpt/mozilla", + "!/tests/wpt/webgl", + "!/tests/wpt/web-platform-tests", + "/tests/wpt/web-platform-tests/tools", +] def linux_tidy_unit(): - return decision.create_task( - task_name="Linux x86_64: tidy + dev build + unit tests", - script=""" - ./mach test-tidy --no-progress --all - ./mach build --dev - ./mach test-unit - ./mach package --dev - ./mach test-tidy --no-progress --self-test - python2.7 ./etc/memory_reports_over_time.py --test - python3 ./etc/taskcluster/mock.py - ./etc/ci/lockfile_changed.sh - ./etc/ci/check_no_panic.sh - """, - **linux_build_kwargs - ) + return linux_build_task("Linux x86_64: tidy + dev build + unit tests").with_script(""" + ./mach test-tidy --no-progress --all + ./mach build --dev + ./mach test-unit + ./mach package --dev + ./mach test-tidy --no-progress --self-test + python2.7 ./etc/memory_reports_over_time.py --test + python3 ./etc/taskcluster/mock.py + ./etc/ci/lockfile_changed.sh + ./etc/ci/check_no_panic.sh + """).create() def with_rust_nightly(): - return decision.create_task( - task_name="Linux x86_64: with Rust Nightly", - script=""" - echo "nightly" > rust-toolchain - ./mach build --dev - ./mach test-unit - """, - **linux_build_kwargs - ) + return linux_build_task("Linux x86_64: with Rust Nightly").with_script(""" + echo "nightly" > rust-toolchain + ./mach build --dev + ./mach test-unit + """).create() def android_arm32(): - return decision.find_or_create_task( - index_bucket="build.android_armv7_release", - index_key=os.environ["GIT_SHA"], # Set in .taskcluster.yml - index_expiry=build_artifacts_expiry, - - task_name="Android ARMv7: build", + return ( + linux_build_task("Android ARMv7: build") # file: NDK parses $(file $SHELL) to tell x86_64 from x86 # wget: servo-media-gstreamer’s build script - script=""" + .with_script(""" apt-get install -y --no-install-recommends openjdk-8-jdk-headless file wget ./etc/ci/bootstrap-android-and-accept-licences.sh ./mach build --android --release - """, - artifacts=[ + """) + .with_artifacts( "/repo/target/armv7-linux-androideabi/release/servoapp.apk", "/repo/target/armv7-linux-androideabi/release/servoview.aar", - ], - **linux_build_kwargs + ) + .find_or_create("build.android_armv7_release." + CONFIG.git_sha) ) def windows_dev(): - python2_task = repack_msi( - url="https://www.python.org/ftp/python/2.7.15/python-2.7.15.amd64.msi", - sha256="5e85f3c4c209de98480acbf2ba2e71a907fd5567a838ad4b6748c76deb286ad7", - ) - gstreamer_task = repack_msi( - url="https://gstreamer.freedesktop.org/data/pkg/windows/" + - "1.14.3/gstreamer-1.0-devel-x86_64-1.14.3.msi", - sha256="b13ea68c1365098c66871f0acab7fd3daa2f2795b5e893fcbb5cd7253f2c08fa", - ) - return decision.create_task( - task_name="Windows x86_64: dev build + unit tests", - script=""" - python -m ensurepip - pip install virtualenv==16.0.0 - - ..\\rustup-init.exe --default-toolchain none -y - - set LIB=%HOMEDRIVE%%HOMEPATH%\\gst\\gstreamer\\1.0\\x86_64\\lib;%LIB% - - call mach.bat build --dev - call mach.bat test-unit - """, - mounts=[ - { - "directory": "git", - "format": "zip", - "content": { - "url": "https://github.com/git-for-windows/git/releases/download/" + - "v2.19.0.windows.1/MinGit-2.19.0-64-bit.zip", - "sha256": "424d24b5fc185a9c5488d7872262464f2facab4f1d4693ea8008196f14a3c19b", - } - }, - { - "directory": "python2", - "format": "zip", - "content": { - "taskId": python2_task, - "artifact": "public/repacked.zip", - } - }, - { - "directory": "gst", - "format": "zip", - "content": { - "taskId": gstreamer_task, - "artifact": "public/repacked.zip", - } - }, - { - "file": "rustup-init.exe", - "content": { - "url": "https://static.rust-lang.org/rustup/archive/" + - "1.13.0/i686-pc-windows-gnu/rustup-init.exe", - "sha256": "43072fbe6b38ab38cd872fa51a33ebd781f83a2d5e83013857fab31fc06e4bf0", - } - } - ], - homedir_path=[ - "git\\cmd", - "python2", - "python2\\Scripts", - ".cargo\\bin", - ], - dependencies=[ - python2_task, - gstreamer_task, - ], - sparse_checkout=[ - "/*", - "!/tests/wpt/metadata", - "!/tests/wpt/mozilla", - "!/tests/wpt/webgl", - "!/tests/wpt/web-platform-tests", - "/tests/wpt/web-platform-tests/tools", - ], - **windows_build_kwargs - ) - - -def repack_msi(url, sha256): - task_definition = dict( - task_name="Windows x86_64: repackage " + url.rpartition("/")[-1], - worker_type="servo-win2016", - with_repo=False, - script=""" - lessmsi x input.msi extracted\\ - cd extracted\\SourceDir - 7za a repacked.zip * - """, - mounts=[ - { - "file": "input.msi", - "content": { - "url": url, - "sha256": sha256, - } - }, - { - "directory": "lessmsi", - "format": "zip", - "content": { - "url": "https://github.com/activescott/lessmsi/releases/download/" + - "v1.6.1/lessmsi-v1.6.1.zip", - "sha256": "540b8801e08ec39ba26a100c855898f455410cecbae4991afae7bb2b4df026c7", - } - }, - { - "directory": "7zip", - "format": "zip", - "content": { - "url": "https://www.7-zip.org/a/7za920.zip", - "sha256": "2a3afe19c180f8373fa02ff00254d5394fec0349f5804e0ad2f6067854ff28ac", - } - } - ], - homedir_path=[ - "lessmsi", - "7zip", - ], - artifacts=[ - "extracted/SourceDir/repacked.zip", - ], - max_run_time_minutes=20, - ) - index_by = json.dumps(task_definition).encode("utf-8") - return decision.find_or_create_task( - index_bucket="by-task-definition", - index_key=hashlib.sha256(index_by).hexdigest(), - index_expiry=build_artifacts_expiry, - **task_definition + return ( + windows_build_task("Windows x86_64: dev build + unit tests") + .with_python2() + .with_rustup() + .with_repacked_msi( + url="https://gstreamer.freedesktop.org/data/pkg/windows/" + + "1.14.3/gstreamer-1.0-devel-x86_64-1.14.3.msi", + sha256="b13ea68c1365098c66871f0acab7fd3daa2f2795b5e893fcbb5cd7253f2c08fa", + path="gst", + ) + .with_script( + "set LIB=%HOMEDRIVE%%HOMEPATH%\\gst\\gstreamer\\1.0\\x86_64\\lib;%LIB%", + "call mach.bat build --dev", + "call mach.bat test-unit", + ) + .create() ) @@ -243,40 +123,23 @@ def linux_wpt(): def linux_release_build(): - return decision.find_or_create_task( - index_bucket="build.linux_x86-64_release", - index_key=os.environ["GIT_SHA"], # Set in .taskcluster.yml - index_expiry=build_artifacts_expiry, - - task_name="Linux x86_64: release build", - script=""" + return ( + linux_build_task("Linux x86_64: release build") + .with_script(""" ./mach build --release --with-debug-assertions -p servo ./etc/ci/lockfile_changed.sh tar -czf /target.tar.gz \ target/release/servo \ target/release/build/osmesa-src-*/output \ target/release/build/osmesa-src-*/out/lib/gallium - """, - artifacts=[ - "/target.tar.gz", - ], - **linux_build_kwargs + """) + .with_artifacts("/target.tar.gz") + .find_or_create("build.linux_x86-64_release." + CONFIG.git_sha) ) def wpt_chunk(release_build_task, total_chunks, this_chunk, extra): - if extra: - name_extra = " + extra" - script_extra = """ - ./mach test-wpt-failure - ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 \ - --log-raw test-wpt-mp.log \ - --log-errorsummary wpt-mp-errorsummary.log \ - eventsource - """ - else: - name_extra = "" - script_extra = "" + name = "Linux x86_64: WPT chunk %s / %s" % (this_chunk, total_chunks) script = """ ./mach test-wpt \ --release \ @@ -296,100 +159,114 @@ def wpt_chunk(release_build_task, total_chunks, this_chunk, extra): # IndexError: list index out of range # File "/repo/python/servo/testing_commands.py", line 533, in filter_intermittents # pull_request = int(last_merge.split(' ')[4][1:]) - create_run_task( - build_task=release_build_task, - task_name="Linux x86_64: WPT chunk %s / %s%s" % (this_chunk, total_chunks, name_extra), - script=script_extra + script, - env={ - "TOTAL_CHUNKS": total_chunks, - "THIS_CHUNK": this_chunk, - }, + if extra: + name += " + extra" + script += """ + ./mach test-wpt-failure + ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 \ + --log-raw test-wpt-mp.log \ + --log-errorsummary wpt-mp-errorsummary.log \ + eventsource + """ + return ( + linux_run_task(name, release_build_task, script) + .with_env(TOTAL_CHUNKS=total_chunks, THIS_CHUNK=this_chunk) + .create() ) -def create_run_task(*, build_task, script, **kwargs): - fetch_build = """ - ./etc/taskcluster/curl-artifact.sh ${BUILD_TASK_ID} target.tar.gz | tar -xz - """ - kwargs.setdefault("env", {})["BUILD_TASK_ID"] = build_task - kwargs.setdefault("dependencies", []).append(build_task) - kwargs.setdefault("artifacts", []).extend( - ("/repo/" + word, log_artifacts_expiry) - for word in script.split() if word.endswith(".log") - ) - return decision.create_task( - script=fetch_build + script, - max_run_time_minutes=60, - dockerfile=dockerfile_path("run"), - **kwargs +def linux_run_task(name, build_task, script): + return ( + linux_task(name) + .with_dockerfile(dockerfile_path("run")) + .with_early_script(""" + ./etc/taskcluster/curl-artifact.sh ${BUILD_TASK_ID} target.tar.gz | tar -xz + """) + .with_env(BUILD_TASK_ID=build_task) + .with_dependencies(build_task) + .with_script(script) + .with_index_and_artifacts_expire_in(log_artifacts_expire_in) + .with_artifacts(*[ + "/repo/" + word + for word in script.split() if word.endswith(".log") + ]) + .with_max_run_time_minutes(60) ) def daily_tasks_setup(): # ':' is not accepted in an index namepspace: # https://docs.taskcluster.net/docs/reference/core/taskcluster-index/references/api - now = decision.now.strftime("%Y-%m-%d_%H-%M-%S") - index_path = "%s.daily.%s" % (decision.index_prefix, now) + now = SHARED.now.strftime("%Y-%m-%d_%H-%M-%S") + index_path = "%s.daily.%s" % (CONFIG.index_prefix, now) # Index this task manually rather than with a route, # so that it is indexed even if it fails. - decision.index_service.insertTask(index_path, { - "taskId": os.environ["TASK_ID"], + SHARED.index_service.insertTask(index_path, { + "taskId": CONFIG.decision_task_id, "rank": 0, "data": {}, - "expires": decision.from_now_json(log_artifacts_expiry), + "expires": SHARED.from_now_json(log_artifacts_expire_in), }) # Unlike when reacting to a GitHub event, # the commit hash is not known until we clone the repository. - os.environ["GIT_SHA"] = \ - subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("utf8").strip() + CONFIG.git_sha_is_current_head() # On failure, notify a few people on IRC # https://docs.taskcluster.net/docs/reference/core/taskcluster-notify/docs/usage notify_route = "notify.irc-channel.#servo.on-failed" - decision.routes_for_all_subtasks.append(notify_route) - decision.scopes_for_all_subtasks.append("queue:route:" + notify_route) - decision.task_name_template = "Servo daily: %s. On failure, ping: " + ping_on_daily_task_failure + CONFIG.routes_for_all_subtasks.append(notify_route) + CONFIG.scopes_for_all_subtasks.append("queue:route:" + notify_route) + CONFIG.task_name_template = "Servo daily: %s. On failure, ping: " + ping_on_daily_task_failure def dockerfile_path(name): return os.path.join(os.path.dirname(__file__), "docker", name + ".dockerfile") -decision = DecisionTask( - task_name_template="Servo: %s", - index_prefix="project.servo.servo", - default_worker_type="servo-docker-worker", - docker_image_cache_expiry=build_dependencies_artifacts_expiry, -) +def linux_task(name): + return DockerWorkerTask(name).with_worker_type("servo-docker-worker") -# https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/caches -cache_scopes = [ - # FIMXE: move to servo-* cache names - "docker-worker:cache:cargo-*", -] -build_caches = { - "cargo-registry-cache": "/root/.cargo/registry", - "cargo-git-cache": "/root/.cargo/git", - "cargo-rustup": "/root/.rustup", - "cargo-sccache": "/root/.cache/sccache", -} -build_kwargs = { - "max_run_time_minutes": 60, -} -linux_build_kwargs = dict(**build_kwargs, **{ - "worker_type": "servo-docker-worker", - "dockerfile": dockerfile_path("build"), - "scopes": cache_scopes, - "cache": build_caches, - "env": linux_build_env, -}) -windows_build_kwargs = dict(**build_kwargs, **{ - "worker_type": "servo-win2016", - "env": build_env, -}) + +def windows_task(name): + return WindowsGenericWorkerTask(name).with_worker_type("servo-win2016") + + +def linux_build_task(name): + return ( + linux_task(name) + # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/caches + # FIMXE: move to servo-* cache names + .with_scopes("docker-worker:cache:cargo-*") + .with_caches(**{ + "cargo-registry-cache": "/root/.cargo/registry", + "cargo-git-cache": "/root/.cargo/git", + "cargo-rustup": "/root/.rustup", + "cargo-sccache": "/root/.cache/sccache", + }) + .with_index_and_artifacts_expire_in(build_artifacts_expire_in) + .with_max_run_time_minutes(60) + .with_dockerfile(dockerfile_path("build")) + .with_env(**build_env, **linux_build_env) + .with_repo() .with_index_and_artifacts_expire_in(build_artifacts_expire_in) + ) + + +def windows_build_task(name): + return ( + windows_task(name) + .with_max_run_time_minutes(60) + .with_env(**build_env, **windows_build_env) + .with_repo(sparse_checkout=windows_sparse_checkout) + ) + + +CONFIG.task_name_template = "Servo: %s" +CONFIG.index_prefix = "project.servo.servo" +CONFIG.docker_images_expire_in = build_dependencies_artifacts_expire_in +CONFIG.repacked_msi_files_expire_in = build_dependencies_artifacts_expire_in if __name__ == "__main__": - main() \ No newline at end of file + main(task_for=os.environ["TASK_FOR"]) \ No newline at end of file diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py index 3e325f3ed44e..2cfbd42d4ee4 100644 --- a/etc/taskcluster/decisionlib.py +++ b/etc/taskcluster/decisionlib.py @@ -19,38 +19,59 @@ import json import os import re +import subprocess import sys import taskcluster -class DecisionTask: +# Public API +__all__ = [ + "CONFIG", "SHARED", "Task", "DockerWorkerTask", + "GenericWorkerTask", "WindowsGenericWorkerTask", +] + + +class Config: """ - Holds some project-specific configuration and provides higher-level functionality - on top of the `taskcluster` package a.k.a. `taskcluster-client.py`. + Global configuration, for users of the library to modify. """ + def __init__(self): + self.task_name_template = "%s" + self.index_prefix = "garbage.servo-decisionlib" + self.scopes_for_all_subtasks = [] + self.routes_for_all_subtasks = [] + self.docker_images_expire_in = "1 month" + self.repacked_msi_files_expire_in = "1 month" + + # Set by docker-worker: + # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/environment + self.decision_task_id = os.environ.get("TASK_ID") - DOCKER_IMAGE_ARTIFACT_FILENAME = "image.tar.lz4" + # Set in the decision task’s payload, such as defined in .taskcluster.yml + self.task_owner = os.environ.get("TASK_OWNER") + self.task_source = os.environ.get("TASK_SOURCE") + self.git_url = os.environ.get("GIT_URL") + self.git_ref = os.environ.get("GIT_REF") + self.git_sha = os.environ.get("GIT_SHA") - # https://github.com/servo/taskcluster-bootstrap-docker-images#image-builder - DOCKER_IMAGE_BUILDER_IMAGE = "servobrowser/taskcluster-bootstrap:image-builder@sha256:" \ - "0a7d012ce444d62ffb9e7f06f0c52fedc24b68c2060711b313263367f7272d9d" + def git_sha_is_current_head(self): + output = subprocess.check_output(["git", "rev-parse", "HEAD"]) + self.git_sha = output.decode("utf8").strip() - def __init__(self, *, index_prefix="garbage.servo-decisionlib", task_name_template="%s", - default_worker_type="github-worker", docker_image_cache_expiry="1 month", - routes_for_all_subtasks=None, scopes_for_all_subtasks=None): - self.task_name_template = task_name_template - self.index_prefix = index_prefix - self.default_worker_type = default_worker_type - self.docker_image_cache_expiry = docker_image_cache_expiry - self.routes_for_all_subtasks = routes_for_all_subtasks or [] - self.scopes_for_all_subtasks = scopes_for_all_subtasks or [] - # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/features#feature-taskclusterproxy - self.queue_service = taskcluster.Queue(options={"baseUrl": "http://taskcluster/queue/v1/"}) - self.index_service = taskcluster.Index(options={"baseUrl": "http://taskcluster/index/v1/"}) +class Shared: + """ + Global shared state. + """ + def __init__(self): self.now = datetime.datetime.utcnow() - self.found_or_created_indices = {} + self.found_or_created_indexed_tasks = {} + + # taskclusterProxy URLs: + # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/features + self.queue_service = taskcluster.Queue(options={"baseUrl": "http://taskcluster/queue/v1/"}) + self.index_service = taskcluster.Index(options={"baseUrl": "http://taskcluster/index/v1/"}) def from_now_json(self, offset): """ @@ -58,231 +79,390 @@ def from_now_json(self, offset): """ return taskcluster.stringDate(taskcluster.fromNow(offset, dateObj=self.now)) - def find_or_create_task(self, *, index_bucket, index_key, index_expiry, artifacts, **kwargs): - """ - Find a task indexed in the given bucket (kind, category, …) and cache key, - on schedule a new one if there isn’t one yet. - Returns the task ID. - """ - index_path = "%s.%s.%s" % (self.index_prefix, index_bucket, index_key) +CONFIG = Config() +SHARED = Shared() +from_now_json = SHARED.from_now_json +now = SHARED.now + + +def chaining(op, attr): + def method(self, *args, **kwargs): + op(self, attr, *args, **kwargs) + return self + return method + + +def append_to_attr(self, attr, *args): getattr(self, attr).extend(args) +def prepend_to_attr(self, attr, *args): getattr(self, attr)[0:0] = list(args) +def update_attr(self, attr, **kwargs): getattr(self, attr).update(kwargs) + + +class Task: + def __init__(self, name): + self.name = name + self.description = "" + self.scheduler_id = "taskcluster-github" + self.provisioner_id = "aws-provisioner-v1" + self.worker_type = "github-worker" + self.deadline_in = "1 day" + self.expires_in = "1 year" + self.index_and_artifacts_expire_in = self.expires_in + self.dependencies = [] + self.scopes = [] + self.routes = [] + self.extra = {} + + with_description = chaining(setattr, "description") + with_scheduler_id = chaining(setattr, "scheduler_id") + with_provisioner_id = chaining(setattr, "provisioner_id") + with_worker_type = chaining(setattr, "worker_type") + with_deadline_in = chaining(setattr, "deadline_in") + with_expires_in = chaining(setattr, "expires_in") + with_index_and_artifacts_expire_in = chaining(setattr, "index_and_artifacts_expire_in") + + with_dependencies = chaining(append_to_attr, "dependencies") + with_scopes = chaining(append_to_attr, "scopes") + with_routes = chaining(append_to_attr, "routes") + + with_extra = chaining(update_attr, "extra") + + def build_worker_payload(self): + raise NotImplementedError + + def create(self): + worker_payload = self.build_worker_payload() + + assert CONFIG.decision_task_id + assert CONFIG.task_owner + assert CONFIG.task_source + queue_payload = { + "taskGroupId": CONFIG.decision_task_id, + "dependencies": [CONFIG.decision_task_id] + self.dependencies, + "schedulerId": self.scheduler_id, + "provisionerId": self.provisioner_id, + "workerType": self.worker_type, + + "created": SHARED.from_now_json(""), + "deadline": SHARED.from_now_json(self.deadline_in), + "expires": SHARED.from_now_json(self.expires_in), + "metadata": { + "name": CONFIG.task_name_template % self.name, + "description": self.description, + "owner": CONFIG.task_owner, + "source": CONFIG.task_source, + }, - task_id = self.found_or_created_indices.get(index_path) + "payload": worker_payload, + } + scopes = self.scopes + CONFIG.scopes_for_all_subtasks + routes = self.routes + CONFIG.routes_for_all_subtasks + if any(r.startswith("index.") for r in routes): + self.extra.setdefault("index", {})["expires"] = \ + SHARED.from_now_json(self.index_and_artifacts_expire_in) + dict_update_if_truthy( + queue_payload, + scopes=scopes, + routes=routes, + extra=self.extra, + ) + + task_id = taskcluster.slugId().decode("utf8") + SHARED.queue_service.createTask(task_id, queue_payload) + print("Scheduled %s" % self.name) + return task_id + + def find_or_create(self, index_path=None): + if not index_path: + worker_type = self.worker_type + index_by = json.dumps([worker_type, self.build_worker_payload()]).encode("utf-8") + index_path = "by-task-definition." + hashlib.sha256(index_by).hexdigest() + index_path = "%s.%s" % (CONFIG.index_prefix, index_path) + + task_id = SHARED.found_or_created_indexed_tasks.get(index_path) if task_id is not None: return task_id try: - result = self.index_service.findTask(index_path) + result = SHARED.index_service.findTask(index_path) task_id = result["taskId"] except taskcluster.TaskclusterRestFailure as e: - if e.status_code == 404: - task_id = self.create_task( - routes=[ - "index." + index_path, - ], - extra={ - "index": { - "expires": self.from_now_json(self.docker_image_cache_expiry), - }, - }, - artifacts=[ - (artifact, index_expiry) - for artifact in artifacts - ], - **kwargs - ) - else: + if e.status_code != 404: raise + self.routes.append("index." + index_path) + task_id = self.create() - self.found_or_created_indices[index_path] = task_id + SHARED.found_or_created_indexed_tasks[index_path] = task_id return task_id - def find_or_build_docker_image(self, dockerfile): - """ - Find a task that built a Docker image based on this `dockerfile`, - or schedule a new image-building task if needed. - Returns the task ID. - """ - dockerfile_contents = expand_dockerfile(dockerfile) - digest = hashlib.sha256(dockerfile_contents).hexdigest() +class GenericWorkerTask(Task): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.max_run_time_minutes = 30 + self.env = {} + self.mounts = [] + self.artifacts = [] - return self.find_or_create_task( - index_bucket="docker-image", - index_key=digest, - index_expiry=self.docker_image_cache_expiry, + with_max_run_time_minutes = chaining(setattr, "max_run_time_minutes") + with_artifacts = chaining(append_to_attr, "artifacts") + with_mounts = chaining(append_to_attr, "mounts") + with_env = chaining(update_attr, "env") - task_name="Docker image: " + image_name(dockerfile), - script=""" - echo "$DOCKERFILE" | docker build -t taskcluster-built - - docker save taskcluster-built | lz4 > /%s - """ % self.DOCKER_IMAGE_ARTIFACT_FILENAME, - env={ - "DOCKERFILE": dockerfile_contents, - }, + def build_command(self): + raise NotImplementedError + + def build_worker_payload(self): + worker_payload = { + "command": self.build_command(), + "maxRunTime": self.max_run_time_minutes * 60 + } + return dict_update_if_truthy( + worker_payload, + env=self.env, + mounts=self.mounts, artifacts=[ - "/" + self.DOCKER_IMAGE_ARTIFACT_FILENAME, + { + "type": "file", + "path": path, + "name": "public/" + url_basename(path), + "expires": SHARED.from_now_json(self.index_and_artifacts_expire_in), + } + for path in self.artifacts ], - max_run_time_minutes=20, - docker_image=self.DOCKER_IMAGE_BUILDER_IMAGE, - features={ - "dind": True, # docker-in-docker - }, - with_repo=False, ) - def create_task(self, *, task_name, script, max_run_time_minutes, - docker_image=None, dockerfile=None, # One of these is required - artifacts=None, dependencies=None, env=None, cache=None, scopes=None, - routes=None, extra=None, features=None, mounts=None, homedir_path=None, - worker_type=None, with_repo=True, sparse_checkout=None): - """ - Schedule a new task. Returns the new task ID. + def _mount_content(self, url_or_artifact_name, task_id, sha256): + if task_id: + content = {"taskId": task_id, "artifact": url_or_artifact_name} + else: + content = {"url": url_or_artifact_name} + if sha256: + content["sha256"] = sha256 + return content + + def with_file_mount(self, url_or_artifact_name, task_id=None, sha256=None, path=None): + return self.with_mounts({ + "file": path or url_basename(url_or_artifact_name), + "content": self._mount_content(url_or_artifact_name, task_id, sha256), + }) + + def with_directory_mount(self, url_or_artifact_name, task_id=None, sha256=None, path=None): + supported_formats = ["rar", "tar.bz2", "tar.gz", "zip"] + for fmt in supported_formats: + suffix = "." + fmt + if url_or_artifact_name.endswith(suffix): + return self.with_mounts({ + "directory": path or url_basename(url_or_artifact_name[:-len(suffix)]), + "content": self._mount_content(url_or_artifact_name, task_id, sha256), + "format": fmt, + }) + raise ValueError( + "%r does not appear to be in one of the supported formats: %r" + % (url_or_artifact_name, ", ".join(supported_formats)) + ) + + +class WindowsGenericWorkerTask(GenericWorkerTask): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.scripts = [] - One of `docker_image` or `dockerfile` (but not both) must be given. - If `dockerfile` is given, the corresponding Docker image is built as needed and cached. + with_script = chaining(append_to_attr, "scripts") + with_early_script = chaining(prepend_to_attr, "scripts") - `with_repo` indicates whether `script` should start in a clone of the git repository. + def build_command(self): + return [deindent(s) for s in self.scripts] + + def with_path_from_homedir(self, *paths): + for p in paths: + self.with_early_script("set PATH=%HOMEDRIVE%%HOMEPATH%\\{};%PATH%".format(p)) + return self + + def with_repo(self, sparse_checkout=None): + git = """ + git init repo + cd repo """ - # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/environment - decision_task_id = os.environ["TASK_ID"] - - dependencies = [decision_task_id] + (dependencies or []) - - # Set in .taskcluster.yml - task_owner = os.environ["TASK_OWNER"] - task_source = os.environ["TASK_SOURCE"] - - env = env or {} - - if with_repo: - # Set in .taskcluster.yml - for k in ["GIT_URL", "GIT_REF", "GIT_SHA"]: - env[k] = os.environ[k] - - worker_type = worker_type or self.default_worker_type - if "docker" in worker_type: - if docker_image and dockerfile: - raise TypeError("cannot use both `docker_image` or `dockerfile`") - if not docker_image and not dockerfile: - raise TypeError("need one of `docker_image` or `dockerfile`") - - if dockerfile: - image_build_task = self.find_or_build_docker_image(dockerfile) - dependencies.append(image_build_task) - docker_image = { - "type": "task-image", - "taskId": image_build_task, - "path": "public/" + self.DOCKER_IMAGE_ARTIFACT_FILENAME, - } + if sparse_checkout: + git += """ + git config core.sparsecheckout true + echo %SPARSE_CHECKOUT_BASE64% > .git\\info\\sparse.b64 + certutil -decode .git\\info\\sparse.b64 .git\\info\\sparse-checkout + type .git\\info\\sparse-checkout + """ + self.env["SPARSE_CHECKOUT_BASE64"] = base64.b64encode( + "\n".join(sparse_checkout).encode("utf-8")) + git += """ + git fetch --depth 1 %GIT_URL% %GIT_REF% + git reset --hard %GIT_SHA% + """ + return self \ + .with_git() \ + .with_script(git) \ + .with_env(**git_env()) + + def with_git(self): + return self \ + .with_path_from_homedir("git\\cmd") \ + .with_directory_mount( + "https://github.com/git-for-windows/git/releases/download/" + + "v2.19.0.windows.1/MinGit-2.19.0-64-bit.zip", + sha256="424d24b5fc185a9c5488d7872262464f2facab4f1d4693ea8008196f14a3c19b", + path="git", + ) - if with_repo: - git = """ - git init repo - cd repo - git fetch --depth 1 "$GIT_URL" "$GIT_REF" - git reset --hard "$GIT_SHA" - """ - script = git + script - command = ["/bin/bash", "--login", "-x", "-e", "-c", deindent(script)] - else: - command = [ - "set PATH=%CD%\\{};%PATH%".format(p) - for p in reversed(homedir_path or []) - ] - if with_repo: - if with_repo: - git = """ - git init repo - cd repo - """ - if sparse_checkout: - git += """ - git config core.sparsecheckout true - echo %SPARSE_CHECKOUT_BASE64% > .git\\info\\sparse.b64 - certutil -decode .git\\info\\sparse.b64 .git\\info\\sparse-checkout - type .git\\info\\sparse-checkout - """ - env["SPARSE_CHECKOUT_BASE64"] = base64.b64encode( - "\n".join(sparse_checkout).encode("utf-8")) - command.append(deindent(git + """ - git fetch --depth 1 %GIT_URL% %GIT_REF% - git reset --hard %GIT_SHA% - """)) - command.append(deindent(script)) + def with_rustup(self): + return self \ + .with_path_from_homedir(".cargo\\bin") \ + .with_early_script( + "%HOMEDRIVE%%HOMEPATH%\\rustup-init.exe --default-toolchain none -y" + ) \ + .with_file_mount( + "https://static.rust-lang.org/rustup/archive/" + + "1.13.0/i686-pc-windows-gnu/rustup-init.exe", + sha256="43072fbe6b38ab38cd872fa51a33ebd781f83a2d5e83013857fab31fc06e4bf0", + ) + def with_repacked_msi(self, url, sha256, path): + repack_task = ( + WindowsGenericWorkerTask("MSI repack: " + url) + .with_worker_type(self.worker_type) + .with_max_run_time_minutes(20) + .with_file_mount(url, sha256=sha256, path="input.msi") + .with_directory_mount( + "https://github.com/activescott/lessmsi/releases/download/" + + "v1.6.1/lessmsi-v1.6.1.zip", + sha256="540b8801e08ec39ba26a100c855898f455410cecbae4991afae7bb2b4df026c7", + path="lessmsi" + ) + .with_directory_mount( + "https://www.7-zip.org/a/7za920.zip", + sha256="2a3afe19c180f8373fa02ff00254d5394fec0349f5804e0ad2f6067854ff28ac", + path="7zip", + ) + .with_path_from_homedir("lessmsi", "7zip") + .with_script(""" + lessmsi x input.msi extracted\\ + cd extracted\\SourceDir + 7za a repacked.zip * + """) + .with_artifacts("extracted/SourceDir/repacked.zip") + .with_index_and_artifacts_expire_in(CONFIG.repacked_msi_files_expire_in) + .find_or_create("repacked-msi." + sha256) + ) + return self \ + .with_dependencies(repack_task) \ + .with_directory_mount("public/repacked.zip", task_id=repack_task, path=path) + + def with_python2(self): + return self \ + .with_repacked_msi( + "https://www.python.org/ftp/python/2.7.15/python-2.7.15.amd64.msi", + sha256="5e85f3c4c209de98480acbf2ba2e71a907fd5567a838ad4b6748c76deb286ad7", + path="python2" + ) \ + .with_early_script(""" + python -m ensurepip + pip install virtualenv==16.0.0 + """) \ + .with_path_from_homedir("python2", "python2\\Scripts") + + + +class DockerWorkerTask(Task): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.docker_image = "ubuntu:bionic-20180821" + self.max_run_time_minutes = 30 + self.scripts = [] + self.env = {} + self.caches = {} + self.features = {} + self.artifacts = [] + + with_docker_image = chaining(setattr, "docker_image") + with_max_run_time_minutes = chaining(setattr, "max_run_time_minutes") + with_artifacts = chaining(append_to_attr, "artifacts") + with_script = chaining(append_to_attr, "scripts") + with_early_script = chaining(prepend_to_attr, "scripts") + with_caches = chaining(update_attr, "caches") + with_env = chaining(update_attr, "env") + + def build_worker_payload(self): worker_payload = { - "maxRunTime": max_run_time_minutes * 60, - "command": command, - "env": env, + "image": self.docker_image, + "maxRunTime": self.max_run_time_minutes * 60, + "command": [ + "/bin/bash", "--login", "-x", "-e", "-c", + deindent("\n".join(self.scripts)) + ], } - if docker_image: - worker_payload["image"] = docker_image - if cache: - worker_payload["cache"] = cache - if features: - worker_payload["features"] = features - if mounts: - worker_payload["mounts"] = mounts - if artifacts: - if "docker" in worker_type: - worker_payload["artifacts"] = { - "public/" + os.path.basename(path): { - "type": "file", - "path": path, - "expires": self.from_now_json(expires), - } - for path, expires in artifacts + return dict_update_if_truthy( + worker_payload, + env=self.env, + cache=self.caches, + features=self.features, + artifacts={ + "public/" + url_basename(path): { + "type": "file", + "path": path, + "expires": SHARED.from_now_json(self.index_and_artifacts_expire_in), } - else: - worker_payload["artifacts"] = [ - { - "type": "file", - "name": "public/" + os.path.basename(path), - "path": path, - "expires": self.from_now_json(expires), - } - for path, expires in artifacts - ] - payload = { - "taskGroupId": decision_task_id, - "dependencies": dependencies or [], - "schedulerId": "taskcluster-github", - "provisionerId": "aws-provisioner-v1", - "workerType": worker_type, - - "created": self.from_now_json(""), - "deadline": self.from_now_json("1 day"), - "metadata": { - "name": self.task_name_template % task_name, - "description": "", - "owner": task_owner, - "source": task_source, + for path in self.artifacts }, - "scopes": (scopes or []) + self.scopes_for_all_subtasks, - "routes": (routes or []) + self.routes_for_all_subtasks, - "extra": extra or {}, - "payload": worker_payload, - } + ) - task_id = taskcluster.slugId().decode("utf8") - self.queue_service.createTask(task_id, payload) - print("Scheduled %s" % task_name) - return task_id + def with_features(self, *names): + self.features.update({name: True for name in names}) + return self + + def with_repo(self): + return self \ + .with_env(**git_env()) \ + .with_early_script(""" + git init repo + cd repo + git fetch --depth 1 "$GIT_URL" "$GIT_REF" + git reset --hard "$GIT_SHA" + """) + + def with_dockerfile(self, dockerfile): + basename = os.path.basename(dockerfile) + suffix = ".dockerfile" + assert basename.endswith(suffix) + image_name = basename[:-len(suffix)] + dockerfile_contents = expand_dockerfile(dockerfile) + digest = hashlib.sha256(dockerfile_contents).hexdigest() -def image_name(dockerfile): - """ - Guess a short name based on the path `dockerfile`. - """ - basename = os.path.basename(dockerfile) - suffix = ".dockerfile" - if basename == "Dockerfile": - return os.path.basename(os.path.dirname(os.path.abspath(dockerfile))) - elif basename.endswith(suffix): - return basename[:-len(suffix)] - else: - return basename + image_build_task = ( + DockerWorkerTask("Docker image: " + image_name) + .with_worker_type(self.worker_type) + .with_max_run_time_minutes(30) + .with_index_and_artifacts_expire_in(CONFIG.docker_images_expire_in) + .with_features("dind") + .with_env(DOCKERFILE=dockerfile_contents) + .with_artifacts("/image.tar.lz4") + .with_script(""" + echo "$DOCKERFILE" | docker build -t taskcluster-built - + docker save taskcluster-built | lz4 > /image.tar.lz4 + """) + .with_docker_image( + # https://github.com/servo/taskcluster-bootstrap-docker-images#image-builder + "servobrowser/taskcluster-bootstrap:image-builder@sha256:" \ + "0a7d012ce444d62ffb9e7f06f0c52fedc24b68c2060711b313263367f7272d9d" + ) + .find_or_create("docker-image." + digest) + ) + + return self \ + .with_dependencies(image_build_task) \ + .with_docker_image({ + "type": "task-image", + "path": "public/image.tar.lz4", + "taskId": image_build_task, + }) def expand_dockerfile(dockerfile): @@ -303,5 +483,26 @@ def expand_dockerfile(dockerfile): return b"\n".join([expand_dockerfile(path), rest]) +def git_env(): + assert CONFIG.git_url + assert CONFIG.git_ref + assert CONFIG.git_sha + return { + "GIT_URL": CONFIG.git_url, + "GIT_REF": CONFIG.git_ref, + "GIT_SHA": CONFIG.git_sha, + } + +def dict_update_if_truthy(d, **kwargs): + for key, value in kwargs.items(): + if value: + d[key] = value + return d + + def deindent(string): - return re.sub("\n +", " \n ", string).strip() + return re.sub("\n +", "\n ", string).strip() + + +def url_basename(url): + return url.rpartition("/")[-1] \ No newline at end of file diff --git a/etc/taskcluster/mock.py b/etc/taskcluster/mock.py index 085c82c29171..02d55beb1a63 100755 --- a/etc/taskcluster/mock.py +++ b/etc/taskcluster/mock.py @@ -32,14 +32,15 @@ def findTask(self, _): Queue = stringDate = fromNow = slugId = MagicMock() sys.modules["taskcluster"] = sys.modules[__name__] sys.dont_write_bytecode = True -code = open(os.path.join(os.path.dirname(__file__), "decision-task.py"), "rb").read() -for k in "TASK_ID TASK_OWNER TASK_SOURCE GIT_URL GIT_REF GIT_SHA".split(): - os.environ[k] = k +os.environ.update(**{k: k for k in "TASK_ID TASK_OWNER TASK_SOURCE GIT_URL GIT_SHA".split()}) +os.environ["GIT_REF"] = "refs/heads/auto" +import decision_task -print("Push:") -os.environ["TASK_FOR"] = "github-push" -exec(code) +print("\n# Push:") +decision_task.main("github-push", mock=True) -print("Daily:") -os.environ["TASK_FOR"] = "daily" -exec(code) +print("\n# Push with hot caches:") +decision_task.main("github-push", mock=True) + +print("\n# Daily:") +decision_task.main("daily", mock=True)