Stores and restores

lazy_build/cache.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import collections
+import os
+import tempfile
+
+import boto3
+import botocore
+
+
+class S3Backend(collections.namedtuple('S3Backend', (
+    'bucket',
+    'path',
+))):
+
+    __slots__ = ()
+
+    @property
+    def s3(self):
+        return boto3.resource('s3')
+
+    def key_for_ctx(self, ctx):
+        return self.path.rstrip('/') + '/' + ctx.hash
+
+    def has_artifact(self, ctx):
+        # what a ridiculous dance we have to do here...
+        try:
+            self.s3.Object(self.bucket, self.key_for_ctx(ctx)).load()
+        except botocore.exceptions.ClientError as ex:
+            if ex.response['Error']['Code'] == '404':
+                return False
+            else:
+                raise
+        else:
+            return True
+
+    def get_artifact(self, ctx):
+        fd, path = tempfile.mkstemp()
+        os.close(fd)
+        self.s3.Bucket(self.bucket).download_file(
+            self.key_for_ctx(ctx),
+            path,
+        )
+        return path
+
+    def store_artifact(self, ctx, path):
+        self.s3.Bucket(self.bucket).upload_file(
+            path,
+            self.key_for_ctx(ctx),
+        )
+
+    def invalidate_artifact(self, ctx):
+        raise NotImplementedError()

lazy_build/cli.py

@@ -3,17 +3,63 @@
 from __future__ import unicode_literals
 
 import argparse
+import json
+import os
+import shlex
+import subprocess
+import sys
 
+from lazy_build import color
 from lazy_build import config
 from lazy_build import context
 
 
+def log(line, **kwargs):
+    kwargs.setdefault('file', sys.stderr)
+    kwargs.setdefault('flush', True)
+    print('{}'.format(line), **kwargs)
+
+
 def build(conf, args):
-    ctx = context.build_context(conf)
+    ctx = context.build_context(conf, args.command)
+
+    if args.verbose:
+        log('Generated build context with hash {}'.format(ctx.hash))
+        log('Individual files:')
+        log(json.dumps(ctx.files, indent=True, sort_keys=True))
+
+    if conf.backend.has_artifact(ctx):
+        log(color.bg_gray('Found remote build artifact, downloading.'))
+        return build_from_artifact(conf, ctx)
+    else:
+        log(color.bg_gray('Found no remote build artifact, building locally.'))
+        return build_from_command(conf, ctx)
+
 
-    import json
-    print(json.dumps(ctx.files, indent=True, sort_keys=True))
-    print('hash:', ctx.hash)
+def build_from_artifact(conf, ctx):
+    log(color.yellow('Downloading artifact...'), end=' ')
+    artifact = conf.backend.get_artifact(ctx)
+    log(color.yellow('done!'))
+    try:
+        log(color.yellow('Extracting artifact...'), end=' ')
+        context.extract_artifact(conf, artifact)
+        log(color.yellow('done!'))
+    finally:
+        os.remove(artifact)
+
+
+def build_from_command(conf, ctx):
+    log(color.yellow('$ ' + ' '.join(shlex.quote(arg) for arg in ctx.command)))
+    subprocess.check_call(ctx.command)
+    log(color.yellow('Packaging artifact...'), end=' ')
+    path = context.package_artifact(conf)
+    log(color.yellow('done!'))
+    try:
+        log(color.yellow('Uploading artifact to shared cache...'), end=' ')
+        conf.backend.store_artifact(ctx, path)
+        log(color.yellow('done!'))
+    finally:
+        os.remove(path)
 
 
 def invalidate(conf, args):
@@ -53,6 +99,9 @@ def main(argv=None):
         '--dry-run', default=False, action='store_true',
         help='say what would be done, without doing it',
     )
+    parser.add_argument(
+        '--verbose', '-v', default=False, action='store_true',
+    )
     parser.add_argument(
         '--action', choices=ACTIONS.keys(), required=True,
         help='action to take',

lazy_build/color.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+"""Stolen from ocflib.misc.shell"""
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import sys
+
+# terminal text color wrappers;
+# this is pretty ugly, but defining them manually lets us avoid hacking flake8
+
+
+def _wrap_colors(color, reset):
+    """Create functions like red('hello') and bg_red('hello') for wrapping
+    strings in ANSI color escapes.
+    >>> red('hello')
+    '\x1b[31mhello\x1b[39m'
+    """
+    def wrapper(string, tty_only=True):
+        """Return colorized string.
+        Takes an optional tty_only argument (defaults to True). When tty_only
+        is set, colors will only be applied if stdout is a tty.  This is useful
+        when you don't want color output (e.g. if redirecting to a file).
+        """
+        if tty_only and not sys.stdout.isatty():
+            return string
+        return '{color}{string}{reset}'.format(
+            color=color,
+            string=string,
+            reset=reset,
+        )
+    return wrapper
+
+
+# Define ANSI color codes
+FG_COLORS = {
+    'black': 30,
+    'red': 31,
+    'green': 32,
+    'yellow': 33,
+    'blue': 34,
+    'magenta': 35,
+    'cyan': 36,
+    'white': 37,
+    'reset': 39,
+    'gray': 90,
+}
+
+BG_COLORS = {k: v + 10 for k, v in FG_COLORS.items()}
+
+
+def code_to_chars(code):
+    """Convert each numeric code to its corresponding characters"""
+    return '\033[' + str(code) + 'm'
+
+
+FG_CODES = {k: code_to_chars(v) for k, v in FG_COLORS.items()}
+BG_CODES = {k: code_to_chars(v) for k, v in BG_COLORS.items()}
+
+
+black = _wrap_colors(FG_CODES['black'], FG_CODES['reset'])
+bg_black = _wrap_colors(BG_CODES['black'], BG_CODES['reset'])
+
+red = _wrap_colors(FG_CODES['red'], FG_CODES['reset'])
+bg_red = _wrap_colors(BG_CODES['red'], BG_CODES['reset'])
+
+green = _wrap_colors(FG_CODES['green'], FG_CODES['reset'])
+bg_green = _wrap_colors(BG_CODES['green'], BG_CODES['reset'])
+
+yellow = _wrap_colors(FG_CODES['yellow'], FG_CODES['reset'])
+bg_yellow = _wrap_colors(BG_CODES['yellow'], BG_CODES['reset'])
+
+blue = _wrap_colors(FG_CODES['blue'], FG_CODES['reset'])
+bg_blue = _wrap_colors(BG_CODES['blue'], BG_CODES['reset'])
+
+magenta = _wrap_colors(FG_CODES['magenta'], FG_CODES['reset'])
+bg_magenta = _wrap_colors(BG_CODES['magenta'], BG_CODES['reset'])
+
+cyan = _wrap_colors(FG_CODES['cyan'], FG_CODES['reset'])
+bg_cyan = _wrap_colors(BG_CODES['cyan'], BG_CODES['reset'])
+
+white = _wrap_colors(FG_CODES['white'], FG_CODES['reset'])
+bg_white = _wrap_colors(BG_CODES['white'], BG_CODES['reset'])
+
+gray = _wrap_colors(FG_CODES['gray'], FG_CODES['reset'])
+bg_gray = _wrap_colors(BG_CODES['gray'], BG_CODES['reset'])
+
+bold = _wrap_colors('\033[1m', '\033[0m')

lazy_build/config.py

@@ -4,11 +4,14 @@
 
 import collections
 
+from lazy_build import cache
+
 
 class Config(collections.namedtuple('Config', (
     'context',
     'ignore',
-    'cache',
+    'output',
+    'backend',
 ))):
 
     __slots__ = ()
@@ -18,19 +21,14 @@ def from_args(cls, args):
         # TODO: this method should also consider config files
 
         # TODO: this
-        cache = CacheConfigS3(
-            bucket='my-cool-bucket',
-            path='/',
+        backend = cache.S3Backend(
+            bucket='some-bucket',
+            path='ckuehl/lazy-build/',
         )
 
         return cls(
-            context=frozenset(args.context or ()),
+            context=frozenset(args.context),
             ignore=frozenset(args.ignore or ()),
-            cache=cache,
+            output=frozenset(args.output),
+            backend=backend,
         )
-
-
-CacheConfigS3 = collections.namedtuple('CacheConfigS3', (
-    'bucket',
-    'path',
-))

lazy_build/context.py

@@ -9,21 +9,28 @@
 import json
 import os
 import os.path
+import shutil
+import tarfile
+import tempfile
 
 
 def hash(data):
     return hashlib.sha256(data).hexdigest()
 
 
 class BuildContext(collections.namedtuple('BuildContext', (
+    'command',
     'files',
 ))):
 
     __slots__ = ()
 
     @property
     def hash(self):
-        return hash(json.dumps(self.files, sort_keys=True).encode('utf8'))
+        return hash(json.dumps(
+            (self.command, self.files),
+            sort_keys=True,
+        ).encode('utf8'))
 
 
 class FileContext(collections.namedtuple('FileContext', (
@@ -38,9 +45,10 @@ def from_path(cls, path):
         assert os.path.lexists(path), path
 
         if os.path.islink(path):
-            # TODO: does this properly handle non-utf8 paths?
-            # (how does Python turn those into strings?)
-            return cls('link', hash(os.readlink(path).encode('utf8')))
+            return cls(
+                'link',
+                hash(os.readlink(path).encode('utf8', 'surrogateescape')),
+            )
         else:
             with open(path, 'rb') as f:
                 return cls('file', hash(f.read()))
@@ -65,7 +73,7 @@ def should_ignore(patterns, path):
     )
 
 
-def build_context(conf):
+def build_context(conf, command):
     ctx = {}
     fringe = {
         os.path.relpath(os.path.realpath(path)) for path in conf.context
@@ -87,4 +95,30 @@ def build_context(conf):
         else:
             ctx[path] = FileContext.from_path(path)
 
-    return BuildContext(files=ctx)
+    return BuildContext(command=command, files=ctx)
+
+
+def package_artifact(conf):
+    fd, tmp = tempfile.mkstemp()
+    os.close(fd)
+    with tarfile.TarFile(tmp, mode='w') as tf:
+        for output_path in conf.output:
+            if os.path.isdir(output_path):
+                for path, _, filenames in os.walk(output_path):
+                    for filename in filenames:
+                        tf.add(os.path.join(path, filename))
+            else:
+                tf.add(output_path)
+    return tmp
+
+
+def extract_artifact(conf, artifact):
+    for output_path in conf.output:
+        if os.path.lexists(output_path):
+            if os.path.isdir(output_path) and not os.path.islink(output_path):
+                shutil.rmtree(output_path)
+            else:
+                os.remove(output_path)
+
+    with tarfile.TarFile(artifact) as tf:
+        tf.extractall()

setup.py

@@ -15,7 +15,7 @@
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.5',
     ],
-    install_requires=[],
+    install_requires=['boto3'],
     packages=find_packages(exclude=('tests*', 'testing*')),
     entry_points={
         'console_scripts': ['lazy-build = lazy_build.cli:main'],

tests/context_test.py

@@ -35,16 +35,20 @@ def test_build_context_simple(tmpdir):
     conf = config.Config(
         context={tmpdir.strpath, tmpdir.join('a').strpath},
         ignore={'d'},
-        cache=None,
+        output=('output',),
+        backend=None,
     )
     tmpdir.join('a').write(b'foo')
     tmpdir.join('b').mkdir()
     tmpdir.join('b/c').write(b'bar')
     tmpdir.join('d').mkdir()
     tmpdir.join('d/e').write(b'baz')
     tmpdir.join('f').mksymlinkto('/etc/passwd')
-    assert context.build_context(conf) == context.BuildContext(files={
-        'a': context.FileContext('file', context.hash(b'foo')),
-        'b/c': context.FileContext('file', context.hash(b'bar')),
-        'f': context.FileContext('link', context.hash(b'/etc/passwd')),
-    })
+    assert context.build_context(conf, 'command') == context.BuildContext(
+        files={
+            'a': context.FileContext('file', context.hash(b'foo')),
+            'b/c': context.FileContext('file', context.hash(b'bar')),
+            'f': context.FileContext('link', context.hash(b'/etc/passwd')),
+        },
+        command='command',
+    )