Move the contrib/build-installer into an invoke generate.installer

tasks/generate.py

@@ -1,7 +1,12 @@
+import base64
 import io
+import os
+import zipfile
 
 import invoke
 
+from . import paths
+
 
 @invoke.task
 def authors():
@@ -26,3 +31,161 @@ def authors():
     with io.open("AUTHORS.txt", "w", encoding="utf8") as fp:
         fp.write(u"\n".join(authors))
         fp.write(u"\n")
+
+
+@invoke.task
+def installer(installer_path=os.path.join(paths.CONTRIB, "get-pip.py")):
+    print("[generate.installer] Generating installer")
+
+    # Define our wrapper script
+    WRAPPER_SCRIPT = b"""
+#!/usr/bin/env python
+#
+# Hi There!
+# You may be wondering what this giant blob of binary data here is, you might
+# even be worried that we're up to something nefarious (good for you for being
+# paranoid!). This is a base64 encoding of a zip file, this zip file contains
+# an entire copy of pip.
+#
+# Pip is a thing that installs packages, pip itself is a package that someone
+# might want to install, especially if they're looking to run this get-pip.py
+# script. Pip has a lot of code to deal with the security of installing
+# packages, various edge cases on various platforms, and other such sort of
+# "tribal knowledge" that has been encoded in its code base. Because of this
+# we basically include an entire copy of pip inside this blob. We do this
+# because the alternatives are attempt to implement a "minipip" that probably
+# doesn't do things correctly and has weird edge cases, or compress pip itself
+# down into a single file.
+#
+# If you're wondering how this is created, it is using an invoke task located
+# in tasks/generate.py called "installer". It can be invoked by using
+# ``invoke generate.installer``.
+
+ZIPFILE = b\"\"\"
+{zipfile}
+\"\"\"
+
+import base64
+import os.path
+import pkgutil
+import shutil
+import sys
+import tempfile
+
+
+def bootstrap(tmpdir=None):
+    # Import pip so we can use it to install pip and maybe setuptools too
+    import pip
+
+    # We always want to install pip
+    packages = ["pip"]
+
+    # Check if the user has requested us not to install setuptools
+    if "--no-setuptools" in sys.argv or os.environ.get("PIP_NO_SETUPTOOLS"):
+        args = [x for x in sys.argv[1:] if x != "--no-setuptools"]
+    else:
+        args = sys.argv[1:]
+
+        # We want to see if setuptools is available before attempting to
+        # install it
+        try:
+            import setuptools  # noqa
+        except ImportError:
+            packages += ["setuptools"]
+
+    delete_tmpdir = False
+    try:
+        # Create a temporary directory to act as a working directory if we were
+        # not given one.
+        if tmpdir is None:
+            tmpdir = tempfile.mkdtemp()
+            delete_tmpdir = True
+
+        # We need to extract the SSL certificates from requests so that they
+        # can be passed to --cert
+        cert_path = os.path.join(tmpdir, "cacert.pem")
+        with open(cert_path, "wb") as cert:
+            cert.write(pkgutil.get_data("pip._vendor.requests", "cacert.pem"))
+
+        # Use an environment variable here so that users can still pass
+        # --cert via sys.argv
+        os.environ.setdefault("PIP_CERT", cert_path)
+
+        # Execute the included pip and use it to install the latest pip and
+        # setuptools from PyPI
+        sys.exit(pip.main(["install", "--upgrade"] + packages + args))
+    finally:
+        # Remove our temporary directory
+        if delete_tmpdir and tmpdir:
+            shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+def main():
+    tmpdir = None
+    try:
+        # Create a temporary working directory
+        tmpdir = tempfile.mkdtemp()
+
+        # Unpack the zipfile into the temporary directory
+        pip_zip = os.path.join(tmpdir, "pip.zip")
+        with open(pip_zip, "wb") as fp:
+            fp.write(base64.decodestring(ZIPFILE))
+
+        # Add the zipfile to sys.path so that we can import it
+        sys.path.insert(0, pip_zip)
+
+        # Run the bootstrap
+        bootstrap(tmpdir=tmpdir)
+    finally:
+        # Clean up our temporary working directory
+        if tmpdir:
+            shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    main()
+""".lstrip()
+
+    # Get all of the files we want to add to the zip file
+    print("[generate.installer] Collect all the files that should be zipped")
+    all_files = []
+    for root, dirs, files in os.walk(os.path.join(paths.PROJECT_ROOT, "pip")):
+        for pyfile in files:
+            if os.path.splitext(pyfile)[1] in {".py", ".pem", ".cfg", ".exe"}:
+                path = os.path.join(root, pyfile)
+                all_files.append(
+                    "/".join(
+                        path.split("/")[len(paths.PROJECT_ROOT.split("/")):]
+                    )
+                )
+
+    # Write the pip files to the zip archive
+    print("[generate.installer] Generate the bundled zip of pip")
+    with zipfile.ZipFile(
+            installer_path, "w", compression=zipfile.ZIP_DEFLATED) as z:
+        for filename in all_files:
+            z.write(os.path.join(paths.PROJECT_ROOT, filename), filename)
+
+    # Get the binary data that compromises our zip file
+    with open(installer_path, "rb") as fp:
+        data = fp.read()
+
+    # Write out the wrapper script that will take the place of the zip script
+    # The reason we need to do this instead of just directly executing the
+    # zip script is that while Python will happily execute a zip script if
+    # passed it on the file system, it will not however allow this to work if
+    # passed it via stdin. This means that this wrapper script is required to
+    # make ``curl https://...../get-pip.py | python`` continue to work.
+    print(
+        "[generate.installer] Write the wrapper script with the bundled zip "
+        "file"
+    )
+    with open(installer_path, "wb") as fp:
+        fp.write(WRAPPER_SCRIPT.format(zipfile=base64.encodestring(data)))
+
+    # Ensure the permissions on the newly created file
+    oldmode = os.stat(installer_path).st_mode & 0o7777
+    newmode = (oldmode | 0o555) & 0o7777
+    os.chmod(installer_path, newmode)
+
+    print("[generate.installer] Generated installer")

tasks/paths.py

@@ -0,0 +1,5 @@
+import os.path
+
+PROJECT_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+
+CONTRIB = os.path.join(PROJECT_ROOT, "contrib")