Skip to content

Commit

Permalink
Auto merge of #42081 - ishitatsuyuki:submodule-better, r=aidanhs
Browse files Browse the repository at this point in the history 
Use the improved submodule handling

r? @alexcrichton

That was a crap...
```
Updating submodules
Traceback (most recent call last):
  File "./x.py", line 20, in <module>
    bootstrap.main()
  File "/home/ishitatsuyuki/Documents/rust/src/bootstrap/bootstrap.py", line 684, in main
    bootstrap()
  File "/home/ishitatsuyuki/Documents/rust/src/bootstrap/bootstrap.py", line 662, in bootstrap
    rb.update_submodules()
  File "/home/ishitatsuyuki/Documents/rust/src/bootstrap/bootstrap.py", line 566, in update_submodules
    path = line[1:].split(' ')[1]
TypeError: a bytes-like object is required, not 'str'
```

Maybe we need to confirm the compatibility of git options, such as `git config` or `git -C` (I believe they existed long before, though). This is tested locally.
  • Loading branch information
@bors
bors committed May 26, 2017
2 parents 256e497 + 15c26c9 commit 5579677
Showing 1 changed file with 78 additions and 86 deletions.
164 changes: 78 additions & 86 deletions src/bootstrap/bootstrap.py
View file
Expand Up @@ -40,7 +40,8 @@ def get(url, path, verbose=False):
return
else:
if verbose:
print("ignoring already-download file " + path + " due to failed verification")
print("ignoring already-download file " +
path + " due to failed verification")
os.unlink(path)
download(temp_path, url, True, verbose)
if not verify(temp_path, sha_path, verbose):
Expand Down Expand Up @@ -100,8 +101,8 @@ def verify(path, sha_path, verbose):
verified = found == expected
if not verified:
print("invalid checksum:\n"
" found: {}\n"
" expected: {}".format(found, expected))
" found: {}\n"
" expected: {}".format(found, expected))
return verified


Expand All @@ -127,20 +128,21 @@ def unpack(tarball, dst, verbose=False, match=None):
shutil.move(tp, fp)
shutil.rmtree(os.path.join(dst, fname))

def run(args, verbose=False, exception=False, cwd=None, env=None):
def run(args, verbose=False, exception=False, **kwargs):
if verbose:
print("running: " + ' '.join(args))
sys.stdout.flush()
# Use Popen here instead of call() as it apparently allows powershell on
# Windows to not lock up waiting for input presumably.
ret = subprocess.Popen(args, cwd=cwd, env=env)
ret = subprocess.Popen(args, **kwargs)
code = ret.wait()
if code != 0:
err = "failed to run: " + ' '.join(args)
if verbose or exception:
raise RuntimeError(err)
sys.exit(err)


def stage0_data(rust_root):
nightlies = os.path.join(rust_root, "src/stage0.txt")
data = {}
Expand All @@ -153,11 +155,13 @@ def stage0_data(rust_root):
data[a] = b
return data


def format_build_time(duration):
return str(datetime.timedelta(seconds=int(duration)))


class RustBuild(object):

def download_stage0(self):
cache_dst = os.path.join(self.build_dir, "cache")
rustc_cache = os.path.join(cache_dst, self.stage0_date())
Expand All @@ -172,11 +176,13 @@ def download_stage0(self):
self.print_what_it_means_to_bootstrap()
if os.path.exists(self.bin_root()):
shutil.rmtree(self.bin_root())
filename = "rust-std-{}-{}.tar.gz".format(rustc_channel, self.build)
filename = "rust-std-{}-{}.tar.gz".format(
rustc_channel, self.build)
url = self._download_url + "/dist/" + self.stage0_date()
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
get("{}/{}".format(url, filename),
tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(),
match="rust-std-" + self.build,
verbose=self.verbose)
Expand All @@ -185,20 +191,25 @@ def download_stage0(self):
url = self._download_url + "/dist/" + self.stage0_date()
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
get("{}/{}".format(url, filename),
tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(),
match="rustc", verbose=self.verbose)
self.fix_executable(self.bin_root() + "/bin/rustc")
self.fix_executable(self.bin_root() + "/bin/rustdoc")
with open(self.rustc_stamp(), 'w') as f:
f.write(self.stage0_date())

if "pc-windows-gnu" in self.build:
filename = "rust-mingw-{}-{}.tar.gz".format(rustc_channel, self.build)
filename = "rust-mingw-{}-{}.tar.gz".format(
rustc_channel, self.build)
url = self._download_url + "/dist/" + self.stage0_date()
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(), match="rust-mingw", verbose=self.verbose)
get("{}/{}".format(url, filename),
tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(),
match="rust-mingw", verbose=self.verbose)

if self.cargo().startswith(self.bin_root()) and \
(not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
Expand All @@ -207,8 +218,10 @@ def download_stage0(self):
url = self._download_url + "/dist/" + self.stage0_date()
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
get("{}/{}".format(url, filename),
tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(),
match="cargo", verbose=self.verbose)
self.fix_executable(self.bin_root() + "/bin/cargo")
with open(self.cargo_stamp(), 'w') as f:
f.write(self.stage0_date())
Expand All @@ -218,7 +231,8 @@ def fix_executable(self, fname):

default_encoding = sys.getdefaultencoding()
try:
ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
ostype = subprocess.check_output(
['uname', '-s']).strip().decode(default_encoding)
except (subprocess.CalledProcessError, WindowsError):
return

Expand All @@ -234,7 +248,8 @@ def fix_executable(self, fname):
print("info: you seem to be running NixOS. Attempting to patch " + fname)

try:
interpreter = subprocess.check_output(["patchelf", "--print-interpreter", fname])
interpreter = subprocess.check_output(
["patchelf", "--print-interpreter", fname])
interpreter = interpreter.strip().decode(default_encoding)
except subprocess.CalledProcessError as e:
print("warning: failed to call patchelf: %s" % e)
Expand All @@ -243,7 +258,8 @@ def fix_executable(self, fname):
loader = interpreter.split("/")[-1]

try:
ldd_output = subprocess.check_output(['ldd', '/run/current-system/sw/bin/sh'])
ldd_output = subprocess.check_output(
['ldd', '/run/current-system/sw/bin/sh'])
ldd_output = ldd_output.strip().decode(default_encoding)
except subprocess.CalledProcessError as e:
print("warning: unable to call ldd: %s" % e)
Expand All @@ -261,7 +277,8 @@ def fix_executable(self, fname):
correct_interpreter = loader_path + loader

try:
subprocess.check_output(["patchelf", "--set-interpreter", correct_interpreter, fname])
subprocess.check_output(
["patchelf", "--set-interpreter", correct_interpreter, fname])
except subprocess.CalledProcessError as e:
print("warning: failed to call patchelf: %s" % e)
return
Expand Down Expand Up @@ -371,16 +388,16 @@ def build_bootstrap(self):
env["CARGO_TARGET_DIR"] = build_dir
env["RUSTC"] = self.rustc()
env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
(os.pathsep + env["LD_LIBRARY_PATH"]) \
if "LD_LIBRARY_PATH" in env else ""
(os.pathsep + env["LD_LIBRARY_PATH"]) \
if "LD_LIBRARY_PATH" in env else ""
env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
(os.pathsep + env["DYLD_LIBRARY_PATH"]) \
if "DYLD_LIBRARY_PATH" in env else ""
(os.pathsep + env["DYLD_LIBRARY_PATH"]) \
if "DYLD_LIBRARY_PATH" in env else ""
env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
(os.pathsep + env["LIBRARY_PATH"]) \
if "LIBRARY_PATH" in env else ""
(os.pathsep + env["LIBRARY_PATH"]) \
if "LIBRARY_PATH" in env else ""
env["PATH"] = os.path.join(self.bin_root(), "bin") + \
os.pathsep + env["PATH"]
os.pathsep + env["PATH"]
if not os.path.isfile(self.cargo()):
raise Exception("no cargo executable found at `%s`" % self.cargo())
args = [self.cargo(), "build", "--manifest-path",
Expand All @@ -395,16 +412,6 @@ def build_bootstrap(self):
args.append("--frozen")
run(args, env=env, verbose=self.verbose)

def output(self, args, env=None, cwd=None):
default_encoding = sys.getdefaultencoding()
proc = subprocess.Popen(args, stdout=subprocess.PIPE, env=env, cwd=cwd)
(out, err) = proc.communicate()
ret = proc.wait()
if ret != 0:
print(out)
sys.exit(ret)
return out.decode(default_encoding)

def build_triple(self):
default_encoding = sys.getdefaultencoding()
config = self.get_toml('build')
Expand All @@ -414,8 +421,10 @@ def build_triple(self):
if config:
return config
try:
ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding)
ostype = subprocess.check_output(
['uname', '-s']).strip().decode(default_encoding)
cputype = subprocess.check_output(
['uname', '-m']).strip().decode(default_encoding)
except (subprocess.CalledProcessError, OSError):
if sys.platform == 'win32':
return 'x86_64-pc-windows-msvc'
Expand All @@ -427,7 +436,8 @@ def build_triple(self):
# The goal here is to come up with the same triple as LLVM would,
# at least for the subset of platforms we're willing to target.
if ostype == 'Linux':
os_from_sp = subprocess.check_output(['uname', '-o']).strip().decode(default_encoding)
os_from_sp = subprocess.check_output(
['uname', '-o']).strip().decode(default_encoding)
if os_from_sp == 'Android':
ostype = 'linux-android'
else:
Expand All @@ -451,7 +461,7 @@ def build_triple(self):
# must be used instead.
try:
cputype = subprocess.check_output(['isainfo',
'-k']).strip().decode(default_encoding)
'-k']).strip().decode(default_encoding)
except (subprocess.CalledProcessError, OSError):
err = "isainfo not found"
if self.verbose:
Expand Down Expand Up @@ -544,51 +554,29 @@ def build_triple(self):

def update_submodules(self):
if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
self.get_toml('submodules') == "false" or \
self.get_mk('CFG_DISABLE_MANAGE_SUBMODULES') == "1":
self.get_toml('submodules') == "false" or \
self.get_mk('CFG_DISABLE_MANAGE_SUBMODULES') == "1":
return

print('Updating submodules')
output = self.output(["git", "submodule", "status"], cwd=self.rust_root)
submodules = []
for line in output.splitlines():
# NOTE `git submodule status` output looks like this:
#
# -5066b7dcab7e700844b0e2ba71b8af9dc627a59b src/liblibc
# +b37ef24aa82d2be3a3cc0fe89bf82292f4ca181c src/compiler-rt (remotes/origin/..)
# e058ca661692a8d01f8cf9d35939dfe3105ce968 src/jemalloc (3.6.0-533-ge058ca6)
#
# The first character can be '-', '+' or ' ' and denotes the
# `State` of the submodule Right next to this character is the
# SHA-1 of the submodule HEAD And after that comes the path to the
# submodule
path = line[1:].split(' ')[1]
submodules.append([path, line[0]])

run(["git", "submodule", "sync"], cwd=self.rust_root)

for submod in submodules:
path, status = submod
if path.endswith('llvm') and \
(self.get_toml('llvm-config') or self.get_mk('CFG_LLVM_ROOT')):
continue
if path.endswith('jemalloc') and \
(self.get_toml('jemalloc') or self.get_mk('CFG_JEMALLOC_ROOT')):
continue
submod_path = os.path.join(self.rust_root, path)

if status == ' ':
run(["git", "reset", "--hard"], cwd=submod_path)
run(["git", "clean", "-fdx"], cwd=submod_path)
elif status == '+':
run(["git", "submodule", "update", path], cwd=self.rust_root)
run(["git", "reset", "--hard"], cwd=submod_path)
run(["git", "clean", "-fdx"], cwd=submod_path)
elif status == '-':
run(["git", "submodule", "init", path], cwd=self.rust_root)
run(["git", "submodule", "update", path], cwd=self.rust_root)
else:
raise ValueError('unknown submodule status: ' + status)
default_encoding = sys.getdefaultencoding()
run(["git", "submodule", "-q", "sync"], cwd=self.rust_root)
submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
["git", "config", "--file", os.path.join(self.rust_root, ".gitmodules"),
"--get-regexp", "path"]
).decode(default_encoding).splitlines()]
submodules = [module for module in submodules
if not ((module.endswith("llvm") and
(self.get_toml('llvm-config') or self.get_mk('CFG_LLVM_ROOT'))) or
(module.endswith("jemalloc") and
(self.get_toml('jemalloc') or self.get_mk('CFG_JEMALLOC_ROOT'))))
]
run(["git", "submodule", "update",
"--init"] + submodules, cwd=self.rust_root, verbose=self.verbose)
run(["git", "submodule", "-q", "foreach", "git",
"reset", "-q", "--hard"], cwd=self.rust_root, verbose=self.verbose)
run(["git", "submodule", "-q", "foreach", "git",
"clean", "-qdfx"], cwd=self.rust_root, verbose=self.verbose)


def bootstrap():
parser = argparse.ArgumentParser(description='Build rust')
Expand Down Expand Up @@ -641,7 +629,7 @@ def bootstrap():
if rb.use_vendored_sources:
if not os.path.exists('.cargo'):
os.makedirs('.cargo')
with open('.cargo/config','w') as f:
with open('.cargo/config', 'w') as f:
f.write("""
[source.crates-io]
replace-with = 'vendored-sources'
Expand Down Expand Up @@ -681,21 +669,25 @@ def bootstrap():
env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
run(args, env=env, verbose=rb.verbose)


def main():
start_time = time()
help_triggered = ('-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
help_triggered = (
'-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
try:
bootstrap()
if not help_triggered:
print("Build completed successfully in %s" % format_build_time(time() - start_time))
print("Build completed successfully in %s" %
format_build_time(time() - start_time))
except (SystemExit, KeyboardInterrupt) as e:
if hasattr(e, 'code') and isinstance(e.code, int):
exit_code = e.code
else:
exit_code = 1
print(e)
if not help_triggered:
print("Build completed unsuccessfully in %s" % format_build_time(time() - start_time))
print("Build completed unsuccessfully in %s" %
format_build_time(time() - start_time))
sys.exit(exit_code)

if __name__ == '__main__':
Expand Down

0 comments on commit 5579677

Please sign in to comment.