Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use Salt for mach bootstrap #14974

Merged
merged 5 commits into from Jan 20, 2017
Merged
Changes from 1 commit
Commits
File filter...
Filter file types
Jump to…
Jump to file
Failed to load files.

Always

Just for now

Create a util.py Python module for common functions

Extracting these functions helps avoid circular dependencies,
and make them easier to find/reuse.
  • Loading branch information
aneeshusa committed Jan 15, 2017
commit 02b054ec9e799705296d5545a744bbf766ddbae6
@@ -8,7 +8,6 @@
# except according to those terms.

from __future__ import print_function, unicode_literals
from socket import error as socket_error

import base64
import json
@@ -17,9 +16,6 @@
import re
import shutil
import sys
import StringIO
import tarfile
import zipfile
import urllib2

from mach.decorators import (
@@ -28,93 +24,8 @@
Command,
)

from servo.command_base import CommandBase, host_triple, BIN_SUFFIX


def download(desc, src, writer, start_byte=0):
if start_byte:
print("Resuming download of %s..." % desc)
else:
print("Downloading %s..." % desc)
dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty())

try:
req = urllib2.Request(src)
if start_byte:
req = urllib2.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)})
resp = urllib2.urlopen(req)

fsize = None
if resp.info().getheader('Content-Length'):
fsize = int(resp.info().getheader('Content-Length').strip()) + start_byte

recved = start_byte
chunk_size = 8192

while True:
chunk = resp.read(chunk_size)
if not chunk:
break
recved += len(chunk)
if not dumb:
if fsize is not None:
pct = recved * 100.0 / fsize
print("\rDownloading %s: %5.1f%%" % (desc, pct), end="")

sys.stdout.flush()
writer.write(chunk)

if not dumb:
print()
except urllib2.HTTPError, e:
print("Download failed (%d): %s - %s" % (e.code, e.reason, src))
if e.code == 403:
print("No Rust compiler binary available for this platform. "
"Please see https://github.com/servo/servo/#prerequisites")
sys.exit(1)
except urllib2.URLError, e:
print("Error downloading Rust compiler: %s. The failing URL was: %s" % (e.reason, src))
sys.exit(1)
except socket_error, e:
print("Looks like there's a connectivity issue, check your Internet connection. %s" % (e))
sys.exit(1)
except KeyboardInterrupt:
writer.flush()
raise


def download_file(desc, src, dst):
tmp_path = dst + ".part"
try:
start_byte = os.path.getsize(tmp_path)
with open(tmp_path, 'ab') as fd:
download(desc, src, fd, start_byte=start_byte)
except os.error:
with open(tmp_path, 'wb') as fd:
download(desc, src, fd)
os.rename(tmp_path, dst)


def download_bytes(desc, src):
content_writer = StringIO.StringIO()
download(desc, src, content_writer)
return content_writer.getvalue()


def extract(src, dst, movedir=None):
if src.endswith(".zip"):
zipfile.ZipFile(src).extractall(dst)
else:
tarfile.open(src).extractall(dst)

if movedir:
for f in os.listdir(movedir):
frm = path.join(movedir, f)
to = path.join(dst, f)
os.rename(frm, to)
os.rmdir(movedir)

os.remove(src)
from servo.command_base import CommandBase, BIN_SUFFIX
from servo.util import download_bytes, download_file, extract, host_triple


@CommandProvider
@@ -3,10 +3,10 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import os
import sys
import shutil
from distutils import spawn

from servo.util import extract, download_file
from base import BaseBootstrapper
from packages import WINDOWS_MSVC as deps

@@ -21,7 +21,6 @@ def ensure_system_packages(self):
self.install_system_packages()

def install_system_packages(self, packages=deps):
from servo.bootstrap_commands import extract, download_file

deps_dir = os.path.join(self.context.sharedir, "msvc-dependencies")
deps_url = "https://servo-rust.s3.amazonaws.com/msvc-deps/"
@@ -24,7 +24,8 @@
Command,
)

from servo.command_base import CommandBase, cd, call, BIN_SUFFIX, host_triple, find_dep_path_newest
from servo.command_base import CommandBase, cd, call, BIN_SUFFIX, find_dep_path_newest
from servo.util import host_triple


def format_duration(seconds):
@@ -18,11 +18,12 @@
from subprocess import PIPE
import sys
import tarfile
import platform

import toml

from mach.registrar import Registrar
from servo.util import host_triple


BIN_SUFFIX = ".exe" if sys.platform == "win32" else ""

@@ -107,51 +108,6 @@ def reset(tarinfo):
os.rename(temp_file, dest_archive)


def host_triple():
os_type = platform.system().lower()
if os_type == "linux":
os_type = "unknown-linux-gnu"
elif os_type == "darwin":
os_type = "apple-darwin"
elif os_type == "android":
os_type = "linux-androideabi"
elif os_type == "windows":
# If we are in a Visual Studio environment, use msvc
if os.getenv("PLATFORM") is not None:
os_type = "pc-windows-msvc"
elif os.getenv("MSYSTEM") is not None:
os_type = "pc-windows-gnu"
else:
os_type = "unknown"
elif os_type.startswith("mingw64_nt-") or os_type.startswith("cygwin_nt-"):
os_type = "pc-windows-gnu"
elif os_type == "freebsd":
os_type = "unknown-freebsd"
else:
os_type = "unknown"

cpu_type = platform.machine().lower()
if os_type.endswith("-msvc"):
# vcvars*.bat should set it properly
platform_env = os.environ.get("PLATFORM")
if platform_env == "X86":
cpu_type = "i686"
elif platform_env == "X64":
cpu_type = "x86_64"
else:
cpu_type = "unknown"
elif cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
cpu_type = "i686"
elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]:
cpu_type = "x86_64"
elif cpu_type == "arm":
cpu_type = "arm"
else:
cpu_type = "unknown"

return "%s-%s" % (cpu_type, os_type)


def normalize_env(env):
# There is a bug in subprocess where it doesn't like unicode types in
# environment variables. Here, ensure all unicode are converted to
@@ -30,8 +30,9 @@

from servo.command_base import (
BuildNotFound, CommandBase,
call, cd, check_call, host_triple, set_osmesa_env,
call, cd, check_call, set_osmesa_env,
)
from servo.util import host_triple

from wptrunner import wptcommandline
from update import updatecommandline
@@ -0,0 +1,151 @@
# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

from __future__ import absolute_import, print_function, unicode_literals

import os
import os.path as path
import platform
import sys
from socket import error as socket_error
import StringIO
import tarfile
import zipfile
import urllib2


def host_triple():
os_type = platform.system().lower()
if os_type == "linux":
os_type = "unknown-linux-gnu"
elif os_type == "darwin":
os_type = "apple-darwin"
elif os_type == "android":
os_type = "linux-androideabi"
elif os_type == "windows":
# If we are in a Visual Studio environment, use msvc
if os.getenv("PLATFORM") is not None:
os_type = "pc-windows-msvc"
elif os.getenv("MSYSTEM") is not None:
os_type = "pc-windows-gnu"
else:
os_type = "unknown"
elif os_type.startswith("mingw64_nt-") or os_type.startswith("cygwin_nt-"):
os_type = "pc-windows-gnu"
elif os_type == "freebsd":
os_type = "unknown-freebsd"
else:
os_type = "unknown"

cpu_type = platform.machine().lower()
if os_type.endswith("-msvc"):
# vcvars*.bat should set it properly
platform_env = os.environ.get("PLATFORM")
if platform_env == "X86":
cpu_type = "i686"
elif platform_env == "X64":
cpu_type = "x86_64"
else:
cpu_type = "unknown"
elif cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
cpu_type = "i686"
elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]:
cpu_type = "x86_64"
elif cpu_type == "arm":
cpu_type = "arm"
else:
cpu_type = "unknown"

return "{}-{}".format(cpu_type, os_type)


def download(desc, src, writer, start_byte=0):
if start_byte:
print("Resuming download of %s..." % desc)
else:
print("Downloading %s..." % desc)
dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty())

try:
req = urllib2.Request(src)
if start_byte:
req = urllib2.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)})
resp = urllib2.urlopen(req)

fsize = None
if resp.info().getheader('Content-Length'):
fsize = int(resp.info().getheader('Content-Length').strip()) + start_byte

recved = start_byte
chunk_size = 8192

while True:
chunk = resp.read(chunk_size)
if not chunk:
break
recved += len(chunk)
if not dumb:
if fsize is not None:
pct = recved * 100.0 / fsize
print("\rDownloading %s: %5.1f%%" % (desc, pct), end="")

sys.stdout.flush()
writer.write(chunk)

if not dumb:
print()
except urllib2.HTTPError, e:
print("Download failed (%d): %s - %s" % (e.code, e.reason, src))
if e.code == 403:
print("No Rust compiler binary available for this platform. "
"Please see https://github.com/servo/servo/#prerequisites")
sys.exit(1)
except urllib2.URLError, e:
print("Error downloading Rust compiler: %s. The failing URL was: %s" % (e.reason, src))
sys.exit(1)
except socket_error, e:
print("Looks like there's a connectivity issue, check your Internet connection. %s" % (e))
sys.exit(1)
except KeyboardInterrupt:
writer.flush()
raise


def download_bytes(desc, src):
content_writer = StringIO.StringIO()
download(desc, src, content_writer)
return content_writer.getvalue()


def download_file(desc, src, dst):
tmp_path = dst + ".part"
try:
start_byte = path.getsize(tmp_path)
with open(tmp_path, 'ab') as fd:
download(desc, src, fd, start_byte=start_byte)
except os.error:
with open(tmp_path, 'wb') as fd:
download(desc, src, fd)
os.rename(tmp_path, dst)


def extract(src, dst, movedir=None):
if src.endswith(".zip"):
zipfile.ZipFile(src).extractall(dst)
else:
tarfile.open(src).extractall(dst)

if movedir:
for f in os.listdir(movedir):
frm = path.join(movedir, f)
to = path.join(dst, f)
os.rename(frm, to)
os.rmdir(movedir)

os.remove(src)
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.