Skip to content
Permalink
Browse files

Fetch repository from artifacts rather than GitHub

  • Loading branch information
SimonSapin committed Nov 22, 2019
1 parent 55a8158 commit 3a8bb531a07aaee60cec110853d1b54d53d438c8
Showing with 48 additions and 40 deletions.
  1. +11 −12 etc/taskcluster/decision_task.py
  2. +37 −28 etc/taskcluster/decisionlib.py
@@ -171,7 +171,7 @@ def linux_tidy_unit_untrusted():
.with_max_run_time_minutes(60)
.with_dockerfile(dockerfile_path("build"))
.with_env(**build_env, **unix_build_env, **linux_build_env)
.with_repo()
.with_repo_bundle()
.with_script("rustup set profile minimal")
# required by components/script_plugins:
.with_script("rustup component add rustc-dev")
@@ -391,7 +391,7 @@ def android_x86_wpt():
.with_capabilities(privileged=True)
.with_scopes("project:servo:docker-worker-kvm:capability:privileged")
.with_dockerfile(dockerfile_path("run-android-emulator"))
.with_repo()
.with_repo_bundle()
)
apk_dir = "target/android/i686-linux-android/release"
return (
@@ -560,7 +560,7 @@ def linux_wpt():
.find_or_create("build.linux_x64_release_w_assertions" + CONFIG.task_id())
)
def linux_run_task(name):
return linux_task(name).with_dockerfile(dockerfile_path("run"))
return linux_task(name).with_dockerfile(dockerfile_path("run")).with_repo_bundle()
wpt_chunks("Linux x64", linux_run_task, release_build_task, repo_dir="/repo",
total_chunks=4, processes=12)

@@ -594,14 +594,14 @@ def update_wpt():
.with_scopes("secrets:get:project/servo/wpt-sync")
.with_index_and_artifacts_expire_in(log_artifacts_expire_in)
.with_max_run_time_minutes(6 * 60)
# Not using the bundle, pushing the new changes to the git remote requires a full repo.
.with_repo(shallow=False, alternate_object_dir="/var/cache/servo.git/objects")
)
return (
with_homebrew(update_task, [
"etc/taskcluster/macos/Brewfile-wpt-update",
"etc/taskcluster/macos/Brewfile",
])
# Pushing the new changes to the git remote requires a full repo clone.
.with_repo(shallow=False, alternate_object_dir="/var/cache/servo.git/objects")
.with_curl_artifact_script(build_task, "target.tar.gz")
.with_script("""
export PKG_CONFIG_PATH="$(brew --prefix libffi)/lib/pkgconfig/"
@@ -637,21 +637,21 @@ def macos_wpt():
priority = "high" if CONFIG.git_ref == "refs/heads/auto" else None
build_task = macos_release_build_with_debug_assertions(priority=priority)
def macos_run_task(name):
task = macos_task(name).with_python2()
task = macos_task(name).with_python2() \
.with_repo_bundle(alternate_object_dir="/var/cache/servo.git/objects")
return with_homebrew(task, ["etc/taskcluster/macos/Brewfile"])
wpt_chunks(
"macOS x64",
macos_run_task,
build_task,
repo_dir="repo",
repo_kwargs=dict(alternate_object_dir="/var/cache/servo.git/objects"),
total_chunks=30,
processes=4,
)


def wpt_chunks(platform, make_chunk_task, build_task, total_chunks, processes,
repo_dir, chunks="all", repo_kwargs={}):
repo_dir, chunks="all"):
if chunks == "all":
chunks = range(total_chunks + 1)
for this_chunk in chunks:
@@ -660,7 +660,6 @@ def wpt_chunks(platform, make_chunk_task, build_task, total_chunks, processes,
this_chunk, total_chunks, width=len(str(total_chunks)),
))
.with_treeherder(platform, "WPT-%s" % this_chunk)
.with_repo(**repo_kwargs)
.with_curl_artifact_script(build_task, "target.tar.gz")
.with_script("tar -xzf target.tar.gz")
.with_index_and_artifacts_expire_in(log_artifacts_expire_in)
@@ -807,7 +806,7 @@ def linux_build_task(name, *, build_env=build_env, install_rustc_dev=True):
.with_max_run_time_minutes(60)
.with_dockerfile(dockerfile_path("build"))
.with_env(**build_env, **unix_build_env, **linux_build_env)
.with_repo()
.with_repo_bundle()
.with_script("rustup set profile minimal")
)
if install_rustc_dev:
@@ -850,7 +849,7 @@ def windows_build_task(name, package=True, arch="x86_64"):
**windows_build_env[arch],
**windows_build_env["all"]
)
.with_repo(sparse_checkout=windows_sparse_checkout)
.with_repo_bundle(sparse_checkout=windows_sparse_checkout)
.with_python2()
.with_directory_mount(
"https://www.python.org/ftp/python/3.7.3/python-3.7.3-embed-amd64.zip",
@@ -903,7 +902,7 @@ def macos_build_task(name):
# https://github.com/servo/servo/issues/24735
.with_max_run_time_minutes(60 * 2)
.with_env(**build_env, **unix_build_env, **macos_build_env)
.with_repo(alternate_object_dir="/var/cache/servo.git/objects")
.with_repo_bundle(alternate_object_dir="/var/cache/servo.git/objects")
.with_python2()
.with_rustup()
# Since macOS workers are long-lived and ~/.rustup kept across tasks:
@@ -58,6 +58,7 @@ def __init__(self):
self.git_url = os.environ.get("GIT_URL")
self.git_ref = os.environ.get("GIT_REF")
self.git_sha = os.environ.get("GIT_SHA")
self.git_bundle_shallow_ref = "refs/heads/shallow"

self.tc_root_url = os.environ.get("TASKCLUSTER_ROOT_URL")
self.default_provisioner_id = "proj-example"
@@ -147,6 +148,9 @@ def __init__(self, name):
self.extra = {}
self.treeherder_required = False
self.priority = None # Defaults to 'lowest'
self.git_fetch_url = CONFIG.git_url
self.git_fetch_ref = CONFIG.git_ref
self.git_checkout_sha = CONFIG.git_sha

# All `with_*` methods return `self`, so multiple method calls can be chained.
with_description = chaining(setattr, "description")
@@ -321,6 +325,14 @@ def with_curl_artifact_script(self, task_id, artifact_name, out_directory=""):
os.path.join(out_directory, url_basename(artifact_name)),
)

def with_repo_bundle(self, **kwargs):
self.git_fetch_url = "../repo.bundle"
self.git_fetch_ref = CONFIG.git_bundle_shallow_ref
self.git_checkout_sha = "FETCH_HEAD"
return self \
.with_curl_artifact_script(CONFIG.decision_task_id, "repo.bundle") \
.with_repo(**kwargs)


class GenericWorkerTask(Task):
"""
@@ -500,13 +512,17 @@ def with_repo(self, sparse_checkout=None, shallow=True):
type .git\\info\\sparse-checkout
"""
git += """
git fetch --no-tags {depth} %GIT_URL% %GIT_REF%
git reset --hard %GIT_SHA%
""".format(depth="--depth 30" if shallow else "")
git fetch --no-tags {depth} {} {}
git reset --hard {}
""".format(
assert_truthy(self.git_fetch_url),
assert_truthy(self.git_fetch_ref),
assert_truthy(self.git_checkout_sha),
depth="--depth 30" if shallow else "",
)
return self \
.with_git() \
.with_script(git) \
.with_env(**git_env())
.with_script(git)

def with_git(self):
"""
@@ -613,7 +629,7 @@ def with_python2(self):


class UnixTaskMixin(Task):
def with_repo(self, shallow=True, alternate_object_dir=None):
def with_repo(self, shallow=True, alternate_object_dir=""):
"""
Make a shallow clone the git repository at the start of the task.
This uses `CONFIG.git_url`, `CONFIG.git_ref`, and `CONFIG.git_sha`
@@ -628,22 +644,19 @@ def with_repo(self, shallow=True, alternate_object_dir=None):
"""
# Not using $GIT_ALTERNATE_OBJECT_DIRECTORIES since it causes
# "object not found - no match for id" errors when Cargo fetches git dependencies
if alternate_object_dir:
self.with_env(ALTERNATE_OBJDIR=alternate_object_dir)
return self \
.with_env(**git_env()) \
.with_early_script("""
.with_script("""
git init repo
cd repo
{alternate}
time git fetch --no-tags {depth} "$GIT_URL" "$GIT_REF"
time git reset --hard "$GIT_SHA"
echo "{alternate}" > .git/objects/info/alternates
time git fetch --no-tags {depth} {} {}
time git reset --hard {}
""".format(
assert_truthy(self.git_fetch_url),
assert_truthy(self.git_fetch_ref),
assert_truthy(self.git_checkout_sha),
depth="--depth 30" if shallow else "",
alternate=(
"""echo "$ALTERNATE_OBJDIR" > .git/objects/info/alternates"""
if alternate_object_dir else ""
)
alternate=alternate_object_dir,
))


@@ -821,15 +834,10 @@ def expand_dockerfile(dockerfile):
return b"\n".join([expand_dockerfile(path), rest])


def git_env():
assert CONFIG.git_url
assert CONFIG.git_ref
assert CONFIG.git_sha
return {
"GIT_URL": CONFIG.git_url,
"GIT_REF": CONFIG.git_ref,
"GIT_SHA": CONFIG.git_sha,
}
def assert_truthy(x):
assert x
return x


def dict_update_if_truthy(d, **kwargs):
for key, value in kwargs.items():
@@ -853,9 +861,10 @@ def make_repo_bundle():
tree = subprocess.check_output(["git", "show", CONFIG.git_sha, "--pretty=%T", "--no-patch"])
message = "Shallow version of commit " + CONFIG.git_sha
commit = subprocess.check_output(["git", "commit-tree", tree.strip(), "-m", message])
subprocess.check_call(["git", "update-ref", "refs/heads/shallow", commit.strip()])
subprocess.check_call(["git", "update-ref", CONFIG.git_bundle_shallow_ref, commit.strip()])
subprocess.check_call(["git", "show-ref"])
with subprocess.Popen(["git", "bundle", "create", "../repo.bundle", "refs/heads/shallow"]) as p:
create = ["git", "bundle", "create", "../repo.bundle", CONFIG.git_bundle_shallow_ref]
with subprocess.Popen(create) as p:
yield
exit_code = p.wait()
if exit_code:

0 comments on commit 3a8bb53

Please sign in to comment.
You can’t perform that action at this time.