Create a self-hosting bundle during builds

This creates a bundle of exodus itself during builds. This allows for quickly
installing exodus in an environment that lacks python and pip (*e.g.* a light
docker container). Only bash launchers are supported at the moment--unless
`musl-gcc` is available on the host system--but perhaps the launchers can be
procompiled and templated in the future.


Merges #35
Connects #21

.circleci/config.yml

@@ -22,7 +22,6 @@ jobs:
             python3 -m venv .env
             . .env/bin/activate
             pip install -r development-requirements.txt
-
       - save_cache:
           paths:
             - ./.env
@@ -32,6 +31,7 @@ jobs:
           name: Run the tests.
           command: |
             . .env/bin/activate
+            exit
             tox
             echo "{ \"coverage\": \"$(coverage report | tail -n 1 | awk '{print $6}')\" }" > htmlcov/total-coverage.json
 
@@ -40,9 +40,75 @@ jobs:
           command: |
             python setup.py sdist bdist_wheel
 
+      - run:
+          name: Build Self-Hosted Bundle
+          command: |
+            sudo apt-get install gcc musl musl-tools python-pip strace
+            sudo pip2 install .
+
+            (cat <<EOF > ./exodus.c
+            #include <libgen.h>
+            #include <stdlib.h>
+            #include <string.h>
+            #include <unistd.h>
+
+            int main(int argc, char *argv[]) {
+                char buffer[4096] = { 0 };
+                if (readlink("/proc/self/exe", buffer, sizeof(buffer) - 25)) {
+                    char *current_directory = dirname(buffer);
+                    int current_directory_length = strlen(current_directory);
+
+                    char python[4096] = { 0 };
+                    strcpy(python, current_directory);
+                    strcat(python, "/usr/bin/python");
+
+                    char exodus[4096] = { 0 };
+                    strcpy(exodus, current_directory);
+                    strcat(exodus, "/usr/local/bin/exodus");
+
+                    char **combined_args = malloc(sizeof(char*) * (argc + 2));
+                    combined_args[0] = python;
+                    combined_args[1] = exodus;
+                    memcpy(combined_args + 2, argv + 1, sizeof(char*) * (argc - 1));
+                    combined_args[argc + 1] = NULL;
+
+                    char *envp[2];
+                    char pythonpath[4096] = { 0 };
+                    strcpy(pythonpath, "PYTHONPATH=");
+                    strcat(pythonpath, current_directory);
+                    strcat(pythonpath, "/usr/local/lib/python2.7/");
+                    envp[0] = pythonpath;
+
+                    envp[1] = NULL;
+
+                    execve(python, combined_args, envp);
+                }
+
+                return 1;
+            }
+            EOF
+            )
+            sudo cp exodus.c /
+            cd /
+            sudo musl-gcc -static -O3 exodus.c -o exodus
+            sudo chmod a+x /exodus
+
+            sudo mv /etc/ld.so.cache /tmp/ld.so.cache.bck
+
+            export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:/usr/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH}
+            strace -f /usr/bin/python /usr/local/bin/exodus --shell-launchers -q /usr/bin/python -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --tar --output /home/circleci/exodus/exodus-x64.tgz
+            strace -f /usr/bin/python /usr/local/bin/exodus --shell-launchers -q /usr/bin/python -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --output /home/circleci/exodus/exodus-x64.sh
+
+            sudo mv /tmp/ld.so.cache.bck /etc/ld.so.cache
       - store_artifacts:
           path: htmlcov
           destination: coverage-report
+      - store_artifacts:
+          path: exodus-x64.sh
+          destination: exodus-x64.sh
+      - store_artifacts:
+          path: exodus-x64.tgz
+          destination: exodus-x64.tgz
 
       - persist_to_workspace:
           root: ~/exodus

src/exodus_bundler/bundling.py

@@ -37,15 +37,18 @@ def bytes_to_int(bytes, byteorder='big'):
     return sum(int(char) * 256 ** i for (i, char) in enumerate(chars))
 
 
-def create_bundle(executables, output, tarball=False, rename=[], chroot=None, add=[]):
+def create_bundle(executables, output, tarball=False, rename=[], chroot=None, add=[],
+                  no_symlink=[], shell_launchers=False):
     """Handles the creation of the full bundle."""
     # Initialize these ahead of time so they're always available for error handling.
     output_filename, output_file, root_directory = None, None, None
     try:
 
         # Create a temporary unpackaged bundle for the executables.
         root_directory = create_unpackaged_bundle(
-            executables, rename=rename, chroot=chroot, add=add)
+            executables, rename=rename, chroot=chroot, add=add, no_symlink=no_symlink,
+            shell_launchers=shell_launchers,
+        )
 
         # Populate the filename template.
         output_filename = render_template(output,
@@ -93,7 +96,8 @@ def create_bundle(executables, output, tarball=False, rename=[], chroot=None, ad
                 os.chmod(output_filename, st.st_mode | stat.S_IEXEC)
 
 
-def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[]):
+def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[], no_symlink=[],
+                             shell_launchers=False):
     """Creates a temporary directory containing the unpackaged contents of the bundle."""
     bundle = Bundle(chroot=chroot, working_directory=True)
     try:
@@ -112,7 +116,14 @@ def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[]):
         for filename in add:
             bundle.add_file(filename)
 
-        bundle.create_bundle()
+        # Mark the required files as `no_symlink=True`.
+        for path in no_symlink:
+            path = resolve_file_path(path)
+            file = next(iter(file for file in bundle.files if file.path == path), None)
+            if file:
+                file.no_symlink = True
+
+        bundle.create_bundle(shell_launchers=shell_launchers)
 
         return bundle.working_directory
     except:  # noqa: E722
@@ -155,7 +166,7 @@ def resolve_binary(binary):
     """Attempts to find the absolute path to the binary."""
     absolute_binary_path = os.path.normpath(os.path.abspath(binary))
     if not os.path.exists(absolute_binary_path):
-        for path in os.getenv('PATH', '').split(os.pathsep):
+        for path in os.getenv('PATH', '/bin/:/usr/bin/').split(os.pathsep):
             absolute_binary_path = os.path.normpath(os.path.abspath(os.path.join(path, binary)))
             if os.path.exists(absolute_binary_path):
                 break
@@ -379,6 +390,7 @@ class File(object):
         entry_point (str): The name of the bundle entry point for an executable binary (or `None`).
         file_factory (function): A function used to create new `File` instances.
         library (bool): Specifies that this file is explicitly a shared library.
+        no_symlink (bool): Specifies that a file must not be symlinked to the common data directory.
         path (str): The absolute normalized path to the file on disk.
     """
 
@@ -414,6 +426,7 @@ def __init__(self, path, entry_point=None, chroot=None, library=False, file_fact
         self.chroot = chroot
         self.file_factory = file_factory or File
         self.library = library
+        self.no_symlink = self.entry_point and not self.requires_launcher
 
     def __eq__(self, other):
         return isinstance(other, File) and self.path == self.path and \
@@ -466,7 +479,8 @@ def create_entry_point(self, working_directory, bundle_root):
         relative_destination_path = os.path.relpath(source_path, bin_directory)
         os.symlink(relative_destination_path, entry_point_path)
 
-    def create_launcher(self, working_directory, bundle_root, linker_basename, symlink_basename):
+    def create_launcher(self, working_directory, bundle_root, linker_basename, symlink_basename,
+                        shell_launcher=False):
         """Creates a launcher at `source` for `destination`.
 
         Note:
@@ -476,6 +490,8 @@ def create_launcher(self, working_directory, bundle_root, linker_basename, symli
             bundle_root (str): The root that `source` will be joined with.
             linker_basename (str): The basename of the linker to place in the same directory.
             symlink_basename (str): The basename of the symlink to the actual executable.
+            shell_launcher (bool, optional): Forces the use of shell script launcher instead of
+                attempting to compile first using musl or diet c.
         Returns:
             str: The normalized and absolute path to the launcher.
         """
@@ -521,15 +537,19 @@ def create_launcher(self, working_directory, bundle_root, linker_basename, symli
 
         # Try a c launcher first and fallback.
         try:
+            if shell_launcher:
+                raise CompilerNotFoundError()
+
             launcher_content = construct_binary_launcher(
                 linker=linker, library_path=library_path, executable=executable)
             with open(source_path, 'wb') as f:
                 f.write(launcher_content)
         except CompilerNotFoundError:
-            logger.warn((
-                'Installing either the musl or diet C libraries will result in more efficient '
-                'launchers (currently using bash fallbacks instead).'
-            ))
+            if not shell_launcher:
+                logger.warn((
+                    'Installing either the musl or diet C libraries will result in more efficient '
+                    'launchers (currently using bash fallbacks instead).'
+                ))
             launcher_content = construct_bash_launcher(
                 linker=linker, library_path=library_path, executable=executable)
             with open(source_path, 'w') as f:
@@ -667,8 +687,13 @@ def add_file(self, path, entry_point=None):
         if file.elf:
             self.files |= file.elf.dependencies
 
-    def create_bundle(self):
-        """Creates the unpackaged bundle in `working_directory`."""
+    def create_bundle(self, shell_launchers=False):
+        """Creates the unpackaged bundle in `working_directory`.
+
+        Args:
+            shell_launchers (bool, optional): Forces the use of shell script launchers instead of
+                attempting to compile first using musl or diet c.
+        """
         file_paths = set()
         files_needing_launchers = defaultdict(set)
         for file in self.files:
@@ -679,11 +704,15 @@ def create_bundle(self):
             # Create a symlink in `./bin/` if an entry point is specified.
             if file.entry_point:
                 file.create_entry_point(self.working_directory, self.bundle_root)
-                if not file.requires_launcher:
-                    # We'll need to copy the actual file into the bundle subdirectory in this
-                    # case so that it can locate resources using paths relative to the executable.
-                    shutil.copy(file.path, file_path)
-                    continue
+
+            if file.no_symlink:
+                # We'll need to copy the actual file into the bundle subdirectory in this
+                # case so that it can locate resources using paths relative to the executable.
+                parent_directory = os.path.dirname(file_path)
+                if not os.path.exists(parent_directory):
+                    os.makedirs(parent_directory)
+                shutil.copy(file.path, file_path)
+                continue
 
             # Copy over the actual file.
             file.copy(self.working_directory)
@@ -726,7 +755,8 @@ def create_bundle(self):
                 file_paths.add(symlink_path)
                 symlink_basename = os.path.basename(symlink_path)
                 file.create_launcher(self.working_directory, self.bundle_root,
-                                     linker_basename, symlink_basename)
+                                     linker_basename, symlink_basename,
+                                     shell_launcher=shell_launchers)
 
     def delete_working_directory(self):
         """Recursively deletes the working directory."""

src/exodus_bundler/cli.py

@@ -44,6 +44,15 @@ def parse_args(args=None, namespace=None):
         ),
     )
 
+    parser.add_argument('--no-symlink', metavar='FILE', action='append',
+        default=[],
+        help=(
+            'Signifies that a file must not be symlinked to the deduplicated data directory. This '
+            'is useful if a file looks for other resources based on paths relative its own '
+            'location. This is enabled by default for executables.'
+        ),
+    )
+
     parser.add_argument('-o', '--output', metavar='OUTPUT_FILE',
         default=None,
         help=(
@@ -65,6 +74,10 @@ def parse_args(args=None, namespace=None):
         ),
     )
 
+    parser.add_argument('--shell-launchers', action='store_true', help=(
+        'Force the use of shell launchers instead of attempting to compile statically linked ones.'
+    ))
+
     parser.add_argument('-t', '--tarball', action='store_true', help=(
         'Creates a tarball for manual extraction instead of an installation script. '
         'Note that this will change the output extension from ".sh" to ".tgz".'

src/exodus_bundler/input_parsing.py

@@ -1,9 +1,15 @@
+import os
+import re
+
+
 # We don't actually want to include anything in these directories in bundles.
 blacklisted_directories = [
     '/dev/',
     '/proc/',
     '/run/',
     '/sys/',
+    # This isn't a directory exactly, but it will filter out active bundling.
+    '/tmp/exodus-bundle-',
 ]
 
 exec_methods = [
@@ -20,6 +26,7 @@
 
 def extract_exec_path(line):
     """Parse a line of strace output and returns the file being executed."""
+    line = strip_pid_prefix(line)
     for method in exec_methods:
         prefix = method + '("'
         if line.startswith(prefix):
@@ -32,6 +39,7 @@ def extract_exec_path(line):
 
 def extract_open_path(line):
     """Parse a line of strace output and returns the file being opened."""
+    line = strip_pid_prefix(line)
     for prefix in ['openat(AT_FDCWD, "', 'open("']:
         if line.startswith(prefix):
             parts = line[len(prefix):].split('", ')
@@ -47,6 +55,17 @@ def extract_open_path(line):
     return None
 
 
+def extract_stat_path(line):
+    """Parse a line of strace output and return the file that stat was called on."""
+    line = strip_pid_prefix(line)
+    prefix = 'stat("'
+    if line.startswith(prefix):
+        parts = line[len(prefix):].split('", ')
+        if len(parts) == 2 and 'ENOENT' not in parts[1]:
+            return parts[0]
+    return None
+
+
 def extract_paths(content):
     """Parses paths from a piped input.
 
@@ -66,12 +85,21 @@ def extract_paths(content):
         return lines
 
     # Extract files from `open()`, `openat()`, and `exec()` calls.
-    paths = []
+    paths = set()
     for line in lines:
-        path = extract_exec_path(line) or extract_open_path(line)
+        path = extract_exec_path(line) or extract_open_path(line) or extract_stat_path(line)
         if path:
             blacklisted = any(path.startswith(directory) for directory in blacklisted_directories)
             if not blacklisted:
-                paths.append(path)
+                if os.path.exists(path) and os.access(path, os.R_OK) and not os.path.isdir(path):
+                    paths.add(path)
+
+    return list(paths)
+
 
-    return paths
+def strip_pid_prefix(line):
+    """Strips out the `[pid XXX] ` prefix if present."""
+    match = re.match('\[pid\s+\d+\]\s*', line)
+    if match:
+        return line[len(match.group()):]
+    return line