Mach Bootstrap: Support for more platforms

.travis.yml

@@ -12,10 +12,10 @@ matrix:
       dist: trusty
       script:
          - ./mach build -d --verbose
+         - ./mach test-compiletest
+         - ./mach test-unit
          - ./mach build-geckolib
          - ./mach test-geckolib
-         - ./mach test-unit
-         - ./mach test-compiletest
          - bash etc/ci/check_no_unwrap.sh
          - bash etc/ci/lockfile_changed.sh
          - bash etc/ci/manifest_changed.sh
@@ -28,18 +28,7 @@ matrix:
       addons:
         apt:
           packages:
-            - cmake
-            - freeglut3-dev
-            - gperf
-            - libosmesa6-dev
-            - libgles2-mesa-dev
-            - python-virtualenv
-            - xorg-dev
             - ccache
-            - libdbus-glib-1-dev
-            - libavformat-dev
-            - libavcodec-dev
-            - libavutil-dev
 
 branches:
   only:

appveyor.yml

@@ -2,7 +2,6 @@ version: 1.0.{build}
 
 environment:
   RUST_BACKTRACE: 1
-  HOME: '%APPVEYOR_BUILD_FOLDER%'
   # The appveyor image we use has a pretty huge set of things installed... we make the
   # initial PATH something sane so we know what to expect
   PATH: "C:\\windows\\system32;\
@@ -69,6 +68,6 @@ install:
 
 build_script:
   - if %BUILD_ENV%==msvc mach build -d -v && mach test-unit
-  - if %BUILD_ENV%==gnu  bash -lc "./mach build -d -v && ./mach test-unit"
+  - if %BUILD_ENV%==gnu  bash -lc "cd $APPVEYOR_BUILD_FOLDER; ./mach build -d -v && ./mach test-unit"
 
 test: off

python/mach_bootstrap.py

@@ -106,6 +106,16 @@ def _activate_virtualenv(topdir):
     script_dir = _get_virtualenv_script_dir()
     activate_path = os.path.join(virtualenv_path, script_dir, "activate_this.py")
     if not (os.path.exists(virtualenv_path) and os.path.exists(activate_path)):
+        # Install missing pip and/or virtualenv packages before creating virtual environment
+        if not _get_exec_path(PIP_NAMES) or not _get_exec_path(VIRTUALENV_NAMES):
+            try:
+                from servo.bootstrapper.bootstrap import Bootstrapper
+
+                bootstrapper = Bootstrapper()
+                bootstrapper.virtualenv()
+            except:
+                pass
+
         virtualenv = _get_exec_path(VIRTUALENV_NAMES)
         if not virtualenv:
             sys.exit("Python virtualenv is not installed. Please install it prior to running mach.")

python/servo/bootstrap_commands.py

@@ -142,11 +142,14 @@ def env(self):
     @CommandArgument('--force', '-f',
                      action='store_true',
                      help='Force reinstall packages')
-    def bootstrap(self, android=False, interactive=False, force=False):
+    @CommandArgument('--silent', '-s',
+                     action='store_true',
+                     help='Run bootstrap without output')
+    def bootstrap(self, android=False, interactive=False, force=False, silent=False):
         from servo.bootstrapper.bootstrap import Bootstrapper
 
         bootstrapper = Bootstrapper()
-        bootstrapper.bootstrap(android=android, interactive=interactive, force=force)
+        bootstrapper.bootstrap(android=android, interactive=interactive, force=force, silent=silent)
 
     @Command('bootstrap-rust',
              description='Download the Rust compiler',

python/servo/bootstrapper/base.py

@@ -4,16 +4,19 @@
 
 from __future__ import print_function, unicode_literals
 
+import os
+import sys
 import distutils
 import subprocess
 
 
 class BaseBootstrapper(object):
     """Base class for system bootstrappers."""
 
-    def __init__(self, interactive=False):
+    def __init__(self, interactive=False, silent=False):
         self.package_manager_updated = False
         self.interactive = interactive
+        self.silent = silent
 
     def ensure_system_packages(self):
         '''
@@ -44,6 +47,27 @@ def which(self, name):
         """
         return distutils.spawn.find_executable(name)
 
+    def run(self, command):
+        if self.silent:
+            FNULL = open(os.devnull, 'w')
+            subprocess.check_call(command, stdout=FNULL, stderr=subprocess.STDOUT)
+        else:
+            subprocess.check_call(command, stdin=sys.stdin)
+
+    def run_check(self, command):
+        return subprocess.call(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    def run_as_root(self, command):
+        if os.geteuid() != 0:
+            if self.which('sudo'):
+                command.insert(0, 'sudo')
+            else:
+                command = ['su', 'root', '-c', ' '.join(command)]
+
+        print('Executing as root:', subprocess.list2cmdline(command))
+
+        self.run(command)
+
     def check_output(self, *args, **kwargs):
         """Run subprocess.check_output."""
         return subprocess.check_output(*args, **kwargs)
@@ -60,3 +84,6 @@ def _update_package_manager(self):
 
         This should be defined in child classes.
         """
+
+    def install_virtualenv(self):
+        """Install virtualenv and pip packages"""

python/servo/bootstrapper/bootstrap.py

@@ -2,14 +2,36 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+# Based on Mozilla's Mozboot: https://dxr.mozilla.org/mozilla-central/source/python/mozboot
+
 from __future__ import print_function
 
+import platform
 import sys
 
+from centosfedora import CentOSFedoraBootstrapper
+from debian import DebianBootstrapper
+from osx import OSXBootstrapper
 from windows_gnu import WindowsGnuBootstrapper
 from windows_msvc import WindowsMsvcBootstrapper
 
 
+DEBIAN_DISTROS = (
+    'Debian',
+    'debian',
+    'Ubuntu',
+    # Most Linux Mint editions are based on Ubuntu. One is based on Debian.
+    # The difference is reported in dist_id from platform.linux_distribution.
+    # But it doesn't matter since we share a bootstrapper between Debian and
+    # Ubuntu.
+    'Mint',
+    'LinuxMint',
+    'Elementary OS',
+    'Elementary',
+    '"elementary OS"',
+)
+
+
 class Bootstrapper(object):
     """Main class that performs system bootstrap."""
 
@@ -18,7 +40,27 @@ def __init__(self):
         cls = None
         args = {}
 
-        if sys.platform.startswith('msys'):
+        if sys.platform.startswith('linux'):
+            distro, version, dist_id = platform.linux_distribution()
+
+            if distro in ('CentOS', 'CentOS Linux', 'Fedora'):
+                cls = CentOSFedoraBootstrapper
+                args['distro'] = distro
+            elif distro in DEBIAN_DISTROS:
+                cls = DebianBootstrapper
+            else:
+                sys.exit('Bootstrap support for this Linux distro not yet available.')
+
+            args['version'] = version
+            args['dist_id'] = dist_id
+
+        elif sys.platform.startswith('darwin'):
+            osx_version = platform.mac_ver()[0]
+
+            cls = OSXBootstrapper
+            args['version'] = osx_version
+
+        elif sys.platform.startswith('msys'):
             cls = WindowsGnuBootstrapper
 
         elif sys.platform.startswith('win32'):
@@ -29,13 +71,22 @@ def __init__(self):
 
         self.instance = cls(**args)
 
-    def bootstrap(self, android=False, interactive=False, force=False):
+    def bootstrap(self, android, interactive, force, silent):
         self.instance.interactive = interactive
         self.instance.force = force
+        self.instance.silent = silent
 
-        if android:
-            self.instance.install_mobile_android_packages()
-        elif force:
+        if force:
             self.instance.install_system_packages()
         else:
             self.instance.ensure_system_packages()
+
+        if android:
+            self.instance.install_mobile_android_packages()
+
+        print
+
+    def virtualenv(self):
+        self.instance.install_virtualenv()
+
+        print

python/servo/bootstrapper/centosfedora.py

@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from base import BaseBootstrapper
+
+
+class CentOSFedoraBootstrapper(BaseBootstrapper):
+    from packages import CENTOSFEDORA as desktop_deps
+
+    def __init__(self, distro, version, dist_id, **kwargs):
+        BaseBootstrapper.__init__(self, **kwargs)
+
+        self.distro = distro
+        self.version = version
+        self.dist_id = dist_id
+
+    def ensure_system_packages(self):
+        install_packages = []
+        installed_list = str(self.check_output(['rpm', '-qa'])).replace('\n', '|')
+        install_packages = []
+        for p in self.desktop_deps:
+            if not "|" + p in installed_list:
+                install_packages += [p]
+        if install_packages:
+            print "Installing missing packages..."
+            self.install_system_packages(install_packages)
+
+    def install_system_packages(self, packages=desktop_deps):
+        self.dnf_install(*packages)
+
+    def install_mobile_android_packages(self):
+        raise NotImplementedError('Bootstrap support for Android not yet available.')
+
+    def install_virtualenv(self):
+        self.dnf_install(*["python-pip", "python-virtualenv"])
+
+    def dnf_install(self, *packages):
+        if self.which('dnf'):
+            command = ['dnf', 'reinstall' if self.force else 'install']
+        else:
+            command = ['yum', 'install']
+
+        if not self.interactive:
+            command.append('-y')
+        command.extend(packages)
+
+        self.run_as_root(command)

python/servo/bootstrapper/debian.py

@@ -0,0 +1,65 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import subprocess
+
+from base import BaseBootstrapper
+
+
+class DebianBootstrapper(BaseBootstrapper):
+    '''Bootstrapper for Debian-based distributions.'''
+
+    from packages import DEBIAN as desktop_deps
+
+    def __init__(self, version, dist_id, **kwargs):
+        BaseBootstrapper.__init__(self, **kwargs)
+        # Find Python virtualenv package name
+        venv = subprocess.Popen(['apt-cache', 'policy', 'virtualenv'], stdout=subprocess.PIPE)
+        self.virtualenv = 'virtualenv' if "virtualenv:" in venv.stdout.read() else 'python-virtualenv'
+        self.desktop_deps += [self.virtualenv]
+
+        self.version = version
+        self.dist_id = dist_id
+
+    def ensure_system_packages(self):
+        printed = False
+
+        for package in self.desktop_deps:
+            if self.run_check(['dpkg', '-s', package]):
+                if not printed:
+                    print "Updating package manager..."
+                    self._ensure_package_manager_updated()
+                    print "Installing missing packages..."
+                    printed = True
+
+                print "Installing %s..." % package
+                self.apt_install(package)
+
+    def install_system_packages(self, packages=desktop_deps):
+        self._ensure_package_manager_updated()
+        self.apt_install(*packages)
+
+    def install_mobile_android_packages(self):
+        raise NotImplementedError('Bootstrap support for Android not yet available.')
+
+    def install_virtualenv(self):
+        self.apt_install(*["python-pip", self.virtualenv])
+
+    def _update_package_manager(self):
+        self.apt_update()
+
+    def apt_install(self, *packages):
+        command = ['apt-get', 'install']
+        if not self.interactive:
+            command.append('-y')
+        if self.force:
+            command.append('--reinstall')
+        command.extend(packages)
+        self.run_as_root(command)
+
+    def apt_update(self):
+        command = ['apt-get', 'update']
+        if not self.interactive:
+            command.append('-y')
+        self.run_as_root(command)