Auto merge of #14039 - metajack:windows-wpt, r=<try>

Fix test-wpt and test-css for Windows.

<!-- Please describe your changes on the following line: -->
In addition to minor changes for Windows, this forces Windows Python to
be used for all Windows builds (instead of using Windows Python only for
pc-windows-msvc builds).

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: -->
- [x] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [ ] These changes fix #__ (github issue number if applicable).

<!-- Either: -->
- [ ] There are tests for these changes OR
- [x] These changes do not require tests because testing the tests is too meta for me

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

<!-- Reviewable:start -->

---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/14039)
<!-- Reviewable:end -->

README.md

@@ -135,16 +135,14 @@ pacman -U --noconfirm $GCC_URL-$GCC_EXT $GCC_URL-ada-$GCC_EXT \
 easy_install-2.7 pip virtualenv
 ```
 
-Open a new MSYS shell window as Administrator and remove the Python binaries (they
-are not compatible with our `mach` driver script yet, unfortunately):
+Add the following line to the end of `.profile` in your home directory:
 
-```sh
-cd /mingw64/bin
-mv python2.exe python2-mingw64.exe
-mv python2.7.exe python2.7-mingw64.exe
+```
+export PATH=/c/Python27:$PATH
 ```
 
-Now, open a MINGW64 (not MSYS!) shell window, and you should be able to build servo as usual!
+Now, open a MINGW64 (not MSYS!) shell window, and you should be able to build
+servo as usual!
 
 #### Cross-compilation for Android
 

appveyor.yml

@@ -68,6 +68,6 @@ install:
 
 build_script:
   - if %BUILD_ENV%==msvc mach build -d -v && mach test-unit
-  - if %BUILD_ENV%==gnu  bash -lc "cd $APPVEYOR_BUILD_FOLDER; ./mach build -d -v && ./mach test-unit"
+  - if %BUILD_ENV%==gnu  bash -lc "export PATH=/c/Python27:$PATH; cd $APPVEYOR_BUILD_FOLDER; ./mach build -d -v && ./mach test-unit"
 
 test: off

etc/ci/buildbot_steps.yml

@@ -97,6 +97,7 @@ arm64:
   - bash ./etc/ci/manifest_changed.sh
 
 windows-dev:
+  - -c set
   - ./mach build --dev
   - ./mach test-unit
   - ./mach build-geckolib

mach

@@ -6,20 +6,87 @@
 # The beginning of this script is both valid shell and valid python,
 # such that the script starts with the shell and is reexecuted with
 # the right python.
-'''which' python2.7 > /dev/null 2> /dev/null && exec python2.7 "$0" "$@" || exec python "$0" "$@"
+''':' && if [ ! -z "$MSYSTEM" ] ; then exec python "$0" "$@" ; else which python2.7 > /dev/null 2> /dev/null && exec python2.7 "$0" "$@" || exec python "$0" "$@" ; fi
 '''
 
 from __future__ import print_function, unicode_literals
 
+
 import os
 import sys
 
+
 def main(args):
     topdir = os.path.abspath(os.path.dirname(sys.argv[0]))
     sys.path.insert(0, os.path.join(topdir, "python"))
     import mach_bootstrap
     mach = mach_bootstrap.bootstrap(topdir)
     sys.exit(mach.run(sys.argv[1:]))
 
+
 if __name__ == '__main__':
+    if sys.platform == 'win32':
+        # This is a complete hack to work around the fact that Windows
+        # multiprocessing needs to import the original module (ie: this
+        # file), but only works if it has a .py extension.
+        #
+        # We do this by a sort of two-level function interposing. The first
+        # level interposes forking.get_command_line() with our version defined
+        # in my_get_command_line(). Our version of get_command_line will
+        # replace the command string with the contents of the fork_interpose()
+        # function to be used in the subprocess.
+        #
+        # The subprocess then gets an interposed imp.find_module(), which we
+        # hack up to find 'mach' without the .py extension, since we already
+        # know where it is (it's us!). If we're not looking for 'mach', then
+        # the original find_module will suffice.
+        #
+        # See also: http://bugs.python.org/issue19946
+        # And: https://bugzilla.mozilla.org/show_bug.cgi?id=914563
+        import inspect
+        from multiprocessing import forking
+        global orig_command_line
+
+        def fork_interpose():
+            import imp
+            import os
+            import sys
+            orig_find_module = imp.find_module
+            def my_find_module(name, dirs):
+                if name == 'mach':
+                    path = os.path.join(dirs[0], 'mach')
+                    f = open(path)
+                    return (f, path, ('', 'r', imp.PY_SOURCE))
+                return orig_find_module(name, dirs)
+
+            # Don't allow writing bytecode file for mach module.
+            orig_load_module = imp.load_module
+            def my_load_module(name, file, path, description):
+                # multiprocess.forking invokes imp.load_module manually and
+                # hard-codes the name __parents_main__ as the module name.
+                if name == '__parents_main__':
+                    old_bytecode = sys.dont_write_bytecode
+                    sys.dont_write_bytecode = True
+                    try:
+                        return orig_load_module(name, file, path, description)
+                    finally:
+                        sys.dont_write_bytecode = old_bytecode
+
+                return orig_load_module(name, file, path, description)
+
+            imp.find_module = my_find_module
+            imp.load_module = my_load_module
+            from multiprocessing.forking import main; main()
+
+        def my_get_command_line():
+            fork_code, lineno = inspect.getsourcelines(fork_interpose)
+            # Remove the first line (for 'def fork_interpose():') and the three
+            # levels of indentation (12 spaces).
+            fork_string = ''.join(x[12:] for x in fork_code[1:])
+            cmdline = orig_command_line()
+            cmdline[2] = fork_string
+            return cmdline
+        orig_command_line = forking.get_command_line
+        forking.get_command_line = my_get_command_line
+
     main(sys.argv)

python/mach_bootstrap.py

@@ -76,9 +76,16 @@
 }
 
 # Possible names of executables
-PYTHON_NAMES = ["python-2.7", "python2.7", "python2", "python"]
-VIRTUALENV_NAMES = ["virtualenv-2.7", "virtualenv2.7", "virtualenv2", "virtualenv"]
-PIP_NAMES = ["pip-2.7", "pip2.7", "pip2", "pip"]
+# NOTE: Windows Python doesn't provide versioned executables, so we must use
+# the plain names. On MSYS, we still use Windows Python.
+if sys.platform in ['msys', 'win32']:
+    PYTHON_NAMES = ["python"]
+    VIRTUALENV_NAMES = ["virtualenv"]
+    PIP_NAMES = ["pip"]
+else:
+    PYTHON_NAMES = ["python-2.7", "python2.7", "python2", "python"]
+    VIRTUALENV_NAMES = ["virtualenv-2.7", "virtualenv2.7", "virtualenv2", "virtualenv"]
+    PIP_NAMES = ["pip-2.7", "pip2.7", "pip2", "pip"]
 
 
 def _get_exec_path(names, is_valid_path=lambda _path: True):
@@ -197,10 +204,11 @@ def bootstrap(topdir):
 
     # We don't support MinGW Python
     if os.path.join(os.sep, 'mingw64', 'bin') in sys.executable:
-        print('Cannot run mach with MinGW Python.')
-        print('\nPlease rename following files:')
-        print(' /mingw64/bin/python2.exe   -> /mingw64/bin/python2-mingw64.exe')
-        print(' /mingw64/bin/python2.7.exe -> /mingw64/bin/python2.7-mingw64.exe')
+        print('Cannot run mach with MinGW or MSYS Python.')
+        print('\nPlease add the path to Windows Python (usually /c/Python27) to your path.')
+        print('You can do this by appending the line:')
+        print('    export PATH=/c/Python27:$PATH')
+        print('to your ~/.profile.')
         sys.exit(1)
 
     # Ensure we are running Python 2.7+. We put this check here so we generate a

python/servo/command_base.py

@@ -107,7 +107,10 @@ def host_triple():
     elif os_type == "android":
         os_type = "linux-androideabi"
     elif os_type == "windows":
-        os_type = "pc-windows-msvc"
+        if os.getenv("MSYSTEM") is None:
+            os_type = "pc-windows-msvc"
+        else:
+            os_type = "pc-windows-gnu"
     elif os_type.startswith("mingw64_nt-") or os_type.startswith("cygwin_nt-"):
         os_type = "pc-windows-gnu"
     elif os_type == "freebsd":

tests/wpt/run.py

@@ -49,7 +49,10 @@ def set_defaults(paths, kwargs):
 
     if kwargs["binary"] is None:
         bin_dir = "release" if kwargs["release"] else "debug"
-        bin_path = servo_path("target", bin_dir, "servo")
+        bin_name = "servo"
+        if sys.platform == "win32":
+            bin_name += ".exe"
+        bin_path = servo_path("target", bin_dir, bin_name)
 
         kwargs["binary"] = bin_path
 

tests/wpt/web-platform-tests/tools/sslutils/openssl.py

@@ -57,7 +57,13 @@ def __call__(self, cmd, *args, **kwargs):
             self.cmd += ["-config", self.conf_path]
         self.cmd += list(args)
 
-        env = os.environ.copy()
+        # Copy the environment, converting to plain strings. Windows
+        # StartProcess is picky about all the keys/values being plain strings,
+        # but at least in MSYS shells, the os.environ dictionary can be mixed.
+        env = {}
+        for k, v in os.environ.iteritems():
+            env[k.encode("utf8")] = v.encode("utf8")
+
         if self.base_conf_path is not None:
             env["OPENSSL_CONF"] = self.base_conf_path.encode("utf8")