From e61a98e93ffbcd87cf4aa2397f552a655b8bf637 Mon Sep 17 00:00:00 2001 From: Loren Carvalho Date: Wed, 2 May 2018 10:45:05 -0500 Subject: [PATCH] Updating docs --- .readthedocs.yml | 6 ++ README.md | 7 +- docs/{api/index.rst => api.rst} | 15 ++--- docs/history.rst | 50 ++++++++++++++ docs/index.rst | 108 ++++++++++++++++++++++++++++-- docs/usage.rst | 4 -- src/shiv/bootstrap/environment.py | 4 ++ src/shiv/bootstrap/interpreter.py | 11 +-- src/shiv/builder.py | 13 +++- src/shiv/cli.py | 2 +- src/shiv/constants.py | 17 ++--- src/shiv/pip.py | 2 +- 12 files changed, 199 insertions(+), 40 deletions(-) create mode 100644 .readthedocs.yml rename docs/{api/index.rst => api.rst} (95%) create mode 100644 docs/history.rst delete mode 100644 docs/usage.rst diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..c8b8b04 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,6 @@ +build: + image: latest + +python: + version: 3.6 + pip_install: true diff --git a/README.md b/README.md index b56e548..4f21686 100644 --- a/README.md +++ b/README.md @@ -85,4 +85,9 @@ tox ### gotchas -Zipapps created with Shiv are not cross-compatible with other architectures. For example, a `pyz` file built on a Mac will only work on other Macs, likewise for RHEL, etc. +Zipapps created with Shiv are not cross-compatible with other architectures. For example, a `pyz` + file built on a Mac will only work on other Macs, likewise for RHEL, etc. + +Zipapps created with Shiv *will* extract themselves into `~/.shiv`, unless overridden via +`SHIV_ROOT`. If you create many utilities with shiv, you may want to ocassionally clean this +directory. diff --git a/docs/api/index.rst b/docs/api.rst similarity index 95% rename from docs/api/index.rst rename to docs/api.rst index 175b07d..bc65870 100644 --- a/docs/api/index.rst +++ b/docs/api.rst @@ -1,9 +1,6 @@ Shiv API ======== -Module contents ---------------- - .. automodule:: shiv :members: :show-inheritance: @@ -15,17 +12,17 @@ cli :members: :show-inheritance: -builder -------- +constants +--- -.. automodule:: shiv.builder +.. automodule:: shiv.constants :members: :show-inheritance: -constants ---------- +builder +------- -.. automodule:: shiv.constants +.. automodule:: shiv.builder :members: :show-inheritance: diff --git a/docs/history.rst b/docs/history.rst new file mode 100644 index 0000000..02daee6 --- /dev/null +++ b/docs/history.rst @@ -0,0 +1,50 @@ +Motivation & Comparisons +======================== + +Why? +---- + +At LinkedIn we ship hundreds of command line utilities to every machine in our data-centers and all +of our employees workstations. The vast majority of these utilties are written in Python. In +addition to these utilities we also have many internal libraries that are uprev'd daily. + +Because of differences in iteration rate and the inherent problems present when dealing with such a +huge dependency graph, we need to package the executables discretely. Initially we took advantage +of the great open source tool `PEX `_. PEX elegantly solved the +isolated packaging requirement we had by including all of a tool's dependencies inside of a single +binary file that we could then distribute! + +However, as our tools matured and picked up additional dependencies, we became acutely aware of the +performance issues being imposed on us by ``pkg_resources``'s +`Issue 510 `_. Since PEX leans heavily on +``pkg_resources`` to bootstrap it's environment, we found ourselves at an impass: lose out on the +ability to neatly package our tools in favor of invocation speed, or impose a few second +performance penalty for the benefit of easy packaging. + +After spending some time investigating extricating pkg_resources from PEX, we decided to start from +a clean slate and thus ``shiv`` was created. + +How? +---- + +Shiv exploits the same features of Python as PEX, packing ``__main__.py`` into a zipfile with a +shebang prepended (akin to zipapps, as defined by +`PEP 441 `_, extracting a dependency directory and +injecting said dependencies at runtime. We have to credit the great work by @wickman, @kwlzn, +@jsirois and the other PEX contributors for laying the groundwork! + +The primary differences between PEX and shiv are: + +* ``shiv`` completey avoids the use of ``pkg_resources``. If it is included by a transitive + dependency, the performance implications are mitigated by limiting the length of ``sys.path`` and + always including the `-s `_ and + `-E `_ Python interpreter flags. +* Instead of shipping our binary with downloaded wheels inside, we package an entire site-packages + directory, as installed by ``pip``. We then bootstrap that directory post-extraction via the + stdlib's ``site.addsitedir`` function. That way, everything works out of the box: namespace + packages, real filesystem access, etc. + +Because we optimize for a shorter ``sys.path`` and don't include ``pkg_resources`` in the critical +path, executales created with ``shiv`` can outperform ones created with PEX by almost 2x. In most +cases the executables created with ``shiv`` are even faster than running a script from within a +virtualenv! diff --git a/docs/index.rst b/docs/index.rst index 6b0a622..3c78cb6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,16 +2,114 @@ shiv 🔪 ==================== Shiv is a command line utility for building fully self -contained Python zipapps as outlined in `Link [PEP 441] ` -but with all their dependencies included. +contained Python zipapps as outlined in `PEP 441 `_ +but with all their dependencies included! -Contents: +Shiv's primary goal is making distributing Python applications fast & easy. + +How it works +------------ + +Shiv includes two major components: a *builder* and a *bootstrap* module. + +Building +^^^^^^^^ + +In order to build self-contained single-artifact executables, shiv leverages ``pip`` and stdlib's +``zipapp`` module. + +.. note:: + Unlike "conventional" zipapps, shiv packs a site-packages style directory of your tool's + dependencies into the resulting binary, and then at bootstrap time extracts it into a ``~/.shiv`` + cache directory. More on this in the `Bootstrapping` section. + +shiv accepts only a few command line parameters of it's own, and any unprocessed parameters are +delegated to ``pip install``. + +For example, if you wanted to create an executable for Pipenv, you'd specify the required +dependencies (``pipenv`` and ``pew``), the callable (either ``-e`` for a setuptools-style entry +point or ``-c`` for a bare console_script name), and the output file. + +.. code-block:: sh + + $ shiv -c pipenv -o ~/bin/pipenv pipenv pew + +This creates an executable (``~/bin/pipenv``) containing all the dependencies required by +``pipenv`` and ``pew`` that invokes the console_script ``pipenv`` when executed! + +You can optionally omit the entry point specification, which will drop you into an interpreter that +is bootstrapped with the dependencies you specify. + +.. code-block:: sh + + $ shiv requests -o requests.pyz --quiet + $ ./requests.pyz + Python 3.6.1 (default, Apr 19 2017, 15:02:08) + [GCC 4.2.1 Compatible Apple LLVM 7.3.0 (clang-703.0.29)] on darwin + Type "help", "copyright", "credits" or "license" for more information. + (InteractiveConsole) + >>> import requests + >>> requests.get('http://shiv.readthedocs.io/') + + +This is particularly useful for running scripts without needing to contaminate your Python +environment, since the ``pyz`` files can be used as a shebang! + +Bootstrapping +^^^^^^^^^^^^^ + +When you run an executable created with shiv a special bootstrap function is called. This function +unpacks dependencies into a uniquely named subdirectory of ``~/.shiv`` and then runs your entry point +(or interactive interpreter) with those dependencies added to your ``sys.path``. Once the +dependencies have been extracted to disk, any further invocations will re-use the 'cached' +site-packages unless they are deleted or moved. + +.. note:: + + Dependencies are extracted (rather than loaded into memory from the zipapp itself) because of + limitations of binary dependencies. Shared objects loaded via the dlopen syscall require a + regular filesystem. Many libraries also expect a filesystem in order to do things like building + paths via ``__file__``, etc. + +Influencing Runtime +------------------- + +There are a number of environment variables you can specify to influence a `pyz` file created with +shiv. + +SHIV_ROOT +^^^^^^^^^ + +This should be populated with a full path, it effectively overrides ``~/.shiv`` as the default base +dir for shiv's extraction cache. + +SHIV_INTERPRETER +^^^^^^^^^^^^^^^^ + +This is a boolean that bypasses and console_script or entry point baked into your pyz. Useful for +dropping into an interactive session in the environment of a built cli utility. + +SHIV_ENTRY_POINT +^^^^^^^^^^^^^^^^ + +This should be populated with a setuptools-style callable, e.g. "module.main:main". This will +execute the pyz with whatever callable entry point you supply. Useful for sharing a single pyz +across many callable 'scripts'. + +SHIV_FORCE_EXTRACT +^^^^^^^^^^^^^^^^^^ + +This forces re-extraction of dependencies even if they've already been extracted. If you make +hotfixes/modifications to the 'cached' dependencies, this will overwrite them. + +Table of Contents +================= .. toctree:: :maxdepth: 2 - usage - api/index + history + api Indices and tables ================== diff --git a/docs/usage.rst b/docs/usage.rst deleted file mode 100644 index b61f525..0000000 --- a/docs/usage.rst +++ /dev/null @@ -1,4 +0,0 @@ -Howto -===== - -todo diff --git a/src/shiv/bootstrap/environment.py b/src/shiv/bootstrap/environment.py index e03c950..6927cfd 100644 --- a/src/shiv/bootstrap/environment.py +++ b/src/shiv/bootstrap/environment.py @@ -1,3 +1,7 @@ +""" +This module contains the ``Environment`` object, which combines settings decided at build time with +overrides defined at runtime (via environment variables). +""" import copy import json import os diff --git a/src/shiv/bootstrap/interpreter.py b/src/shiv/bootstrap/interpreter.py index edd4450..4bc8b80 100644 --- a/src/shiv/bootstrap/interpreter.py +++ b/src/shiv/bootstrap/interpreter.py @@ -1,3 +1,8 @@ +""" +The code in this module is adapted from https://github.com/pantsbuild/pex/blob/master/pex/pex.py + +It is used to enter an interactive interpreter session from an executable created with ``shiv``. +""" import code import sys @@ -13,7 +18,7 @@ def execute_content(name, content): ast = compile(content, name, "exec", flags=0, dont_inherit=1) except SyntaxError: raise RuntimeError( - "Unable to parse {}. Is it a Python script? Syntax correct?".format(name) + f"Unable to parse {name}. Is it a Python script? Syntax correct?" ) old_name, old_file = globals().get("__name__"), globals().get("__file__") @@ -40,9 +45,7 @@ def execute_interpreter(): name, content = sys.argv[1], fp.read() except (FileNotFoundError, IsADirectoryError, PermissionError) as e: raise RuntimeError( - "Could not open {} in the environment [{}]: {}".format( - sys.argv[1], sys.argv[0], e - ) + f"Could not open {sys.argv[1]} in the environment [{sys.argv[0]}]: {e}" ) sys.argv = sys.argv[1:] diff --git a/src/shiv/builder.py b/src/shiv/builder.py index cdf85d8..640a560 100644 --- a/src/shiv/builder.py +++ b/src/shiv/builder.py @@ -1,6 +1,9 @@ -"""This module contains a simplified implementation of Python's "zipapp" module. +""" +This module is a slightly modified implementation of Python's "zipapp" module. + +We've copied a lot of zipapp's code here in order to backport support for compression. +https://docs.python.org/3.7/library/zipapp.html#cmdoption-zipapp-c -We've copied code here in order to patch in support for compression. """ import contextlib import zipfile @@ -55,8 +58,12 @@ def maybe_open(archive: Union[str, Path], mode: str) -> Generator[IO[Any], None, def create_archive( source: Path, target: Path, interpreter: Path, main: str, compressed: bool = True ) -> None: - """Create an application archive from SOURCE.""" + """Create an application archive from SOURCE. + A slightly modified version of stdlib's + `zipapp.create_archive `_ + + """ # Check that main has the right format. mod, sep, fn = main.partition(":") mod_ok = all(part.isidentifier() for part in mod.split(".")) diff --git a/src/shiv/cli.py b/src/shiv/cli.py index e1a8eb7..d95075d 100644 --- a/src/shiv/cli.py +++ b/src/shiv/cli.py @@ -100,7 +100,7 @@ def main( Shiv is a command line utility for building fully self-contained Python zipapps as outlined in PEP 441, but with all their dependencies included! """ - quiet = "-q" in pip_args + quiet = "-q" in pip_args or '--quiet' in pip_args if not quiet: click.secho(" shiv! " + SHIV, bold=True) diff --git a/src/shiv/constants.py b/src/shiv/constants.py index 1bdd659..1b82477 100644 --- a/src/shiv/constants.py +++ b/src/shiv/constants.py @@ -1,3 +1,4 @@ +"""This module contains various error messages.""" from typing import Tuple, Dict # errors: @@ -12,16 +13,8 @@ PIP_INSTALL_ERROR = "\nPip install failed!\n" PIP_REQUIRE_VIRTUALENV = "PIP_REQUIRE_VIRTUALENV" BLACKLISTED_ARGS: Dict[Tuple[str, ...], str] = { - ( - "-t", "--target" - ): "Shiv already supplies a target internally, so overriding is not allowed.", - ( - "--editable", - ): "Editable installs don't actually install via pip (they are just linked), so they are not allowed.", - ( - "-d", "--download" - ): "Shiv needs to actually perform an install, not merely a download.", - ( - "--user", "--root", "--prefix" - ): "Which conflicts with Shiv's internal use of '--target'.", + ("-t", "--target"): "Shiv already supplies a target internally, so overriding is not allowed.", + ("--editable", ): "Editable installs don't actually install via pip (they are just linked), so they are not allowed.", + ("-d", "--download"): "Shiv needs to actually perform an install, not merely a download.", + ("--user", "--root", "--prefix"): "Which conflicts with Shiv's internal use of '--target'.", } diff --git a/src/shiv/pip.py b/src/shiv/pip.py index 9f3a715..06f2062 100644 --- a/src/shiv/pip.py +++ b/src/shiv/pip.py @@ -30,7 +30,7 @@ def install(interpreter_path: str, args: List[str]) -> None: Accepts a list of pip arguments. - .. example:: + .. code-block:: py >>> install('/usr/local/bin/python3', ['numpy', '--target', 'site-packages']) Collecting numpy