diff --git a/lazy_build/cache.py b/lazy_build/cache.py new file mode 100644 index 0000000..01c84ce --- /dev/null +++ b/lazy_build/cache.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import +from __future__ import unicode_literals + +import collections +import os +import tempfile + +import boto3 +import botocore + + +class S3Backend(collections.namedtuple('S3Backend', ( + 'bucket', + 'path', +))): + + __slots__ = () + + @property + def s3(self): + return boto3.resource('s3') + + def key_for_ctx(self, ctx): + return self.path.rstrip('/') + '/' + ctx.hash + + def has_artifact(self, ctx): + # what a ridiculous dance we have to do here... + try: + self.s3.Object(self.bucket, self.key_for_ctx(ctx)).load() + except botocore.exceptions.ClientError as ex: + if ex.response['Error']['Code'] == '404': + return False + else: + raise + else: + return True + + def get_artifact(self, ctx): + fd, path = tempfile.mkstemp() + os.close(fd) + self.s3.Bucket(self.bucket).download_file( + self.key_for_ctx(ctx), + path, + ) + return path + + def store_artifact(self, ctx, path): + self.s3.Bucket(self.bucket).upload_file( + path, + self.key_for_ctx(ctx), + ) + + def invalidate_artifact(self, ctx): + raise NotImplementedError() diff --git a/lazy_build/cli.py b/lazy_build/cli.py index 1660a4b..3de773a 100644 --- a/lazy_build/cli.py +++ b/lazy_build/cli.py @@ -3,17 +3,63 @@ from __future__ import unicode_literals import argparse +import json +import os +import shlex +import subprocess +import sys +from lazy_build import color from lazy_build import config from lazy_build import context +def log(line, **kwargs): + kwargs.setdefault('file', sys.stderr) + kwargs.setdefault('flush', True) + print('{}'.format(line), **kwargs) + + def build(conf, args): - ctx = context.build_context(conf) + ctx = context.build_context(conf, args.command) + + if args.verbose: + log('Generated build context with hash {}'.format(ctx.hash)) + log('Individual files:') + log(json.dumps(ctx.files, indent=True, sort_keys=True)) + + if conf.backend.has_artifact(ctx): + log(color.bg_gray('Found remote build artifact, downloading.')) + return build_from_artifact(conf, ctx) + else: + log(color.bg_gray('Found no remote build artifact, building locally.')) + return build_from_command(conf, ctx) + - import json - print(json.dumps(ctx.files, indent=True, sort_keys=True)) - print('hash:', ctx.hash) +def build_from_artifact(conf, ctx): + log(color.yellow('Downloading artifact...'), end=' ') + artifact = conf.backend.get_artifact(ctx) + log(color.yellow('done!')) + try: + log(color.yellow('Extracting artifact...'), end=' ') + context.extract_artifact(conf, artifact) + log(color.yellow('done!')) + finally: + os.remove(artifact) + + +def build_from_command(conf, ctx): + log(color.yellow('$ ' + ' '.join(shlex.quote(arg) for arg in ctx.command))) + subprocess.check_call(ctx.command) + log(color.yellow('Packaging artifact...'), end=' ') + path = context.package_artifact(conf) + log(color.yellow('done!')) + try: + log(color.yellow('Uploading artifact to shared cache...'), end=' ') + conf.backend.store_artifact(ctx, path) + log(color.yellow('done!')) + finally: + os.remove(path) def invalidate(conf, args): @@ -53,6 +99,9 @@ def main(argv=None): '--dry-run', default=False, action='store_true', help='say what would be done, without doing it', ) + parser.add_argument( + '--verbose', '-v', default=False, action='store_true', + ) parser.add_argument( '--action', choices=ACTIONS.keys(), required=True, help='action to take', diff --git a/lazy_build/color.py b/lazy_build/color.py new file mode 100644 index 0000000..b28b4fd --- /dev/null +++ b/lazy_build/color.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +"""Stolen from ocflib.misc.shell""" +from __future__ import absolute_import +from __future__ import unicode_literals + +import sys + +# terminal text color wrappers; +# this is pretty ugly, but defining them manually lets us avoid hacking flake8 + + +def _wrap_colors(color, reset): + """Create functions like red('hello') and bg_red('hello') for wrapping + strings in ANSI color escapes. + >>> red('hello') + '\x1b[31mhello\x1b[39m' + """ + def wrapper(string, tty_only=True): + """Return colorized string. + Takes an optional tty_only argument (defaults to True). When tty_only + is set, colors will only be applied if stdout is a tty. This is useful + when you don't want color output (e.g. if redirecting to a file). + """ + if tty_only and not sys.stdout.isatty(): + return string + return '{color}{string}{reset}'.format( + color=color, + string=string, + reset=reset, + ) + return wrapper + + +# Define ANSI color codes +FG_COLORS = { + 'black': 30, + 'red': 31, + 'green': 32, + 'yellow': 33, + 'blue': 34, + 'magenta': 35, + 'cyan': 36, + 'white': 37, + 'reset': 39, + 'gray': 90, +} + +BG_COLORS = {k: v + 10 for k, v in FG_COLORS.items()} + + +def code_to_chars(code): + """Convert each numeric code to its corresponding characters""" + return '\033[' + str(code) + 'm' + + +FG_CODES = {k: code_to_chars(v) for k, v in FG_COLORS.items()} +BG_CODES = {k: code_to_chars(v) for k, v in BG_COLORS.items()} + + +black = _wrap_colors(FG_CODES['black'], FG_CODES['reset']) +bg_black = _wrap_colors(BG_CODES['black'], BG_CODES['reset']) + +red = _wrap_colors(FG_CODES['red'], FG_CODES['reset']) +bg_red = _wrap_colors(BG_CODES['red'], BG_CODES['reset']) + +green = _wrap_colors(FG_CODES['green'], FG_CODES['reset']) +bg_green = _wrap_colors(BG_CODES['green'], BG_CODES['reset']) + +yellow = _wrap_colors(FG_CODES['yellow'], FG_CODES['reset']) +bg_yellow = _wrap_colors(BG_CODES['yellow'], BG_CODES['reset']) + +blue = _wrap_colors(FG_CODES['blue'], FG_CODES['reset']) +bg_blue = _wrap_colors(BG_CODES['blue'], BG_CODES['reset']) + +magenta = _wrap_colors(FG_CODES['magenta'], FG_CODES['reset']) +bg_magenta = _wrap_colors(BG_CODES['magenta'], BG_CODES['reset']) + +cyan = _wrap_colors(FG_CODES['cyan'], FG_CODES['reset']) +bg_cyan = _wrap_colors(BG_CODES['cyan'], BG_CODES['reset']) + +white = _wrap_colors(FG_CODES['white'], FG_CODES['reset']) +bg_white = _wrap_colors(BG_CODES['white'], BG_CODES['reset']) + +gray = _wrap_colors(FG_CODES['gray'], FG_CODES['reset']) +bg_gray = _wrap_colors(BG_CODES['gray'], BG_CODES['reset']) + +bold = _wrap_colors('\033[1m', '\033[0m') diff --git a/lazy_build/config.py b/lazy_build/config.py index edee861..ec1adcb 100644 --- a/lazy_build/config.py +++ b/lazy_build/config.py @@ -4,11 +4,14 @@ import collections +from lazy_build import cache + class Config(collections.namedtuple('Config', ( 'context', 'ignore', - 'cache', + 'output', + 'backend', ))): __slots__ = () @@ -18,19 +21,14 @@ def from_args(cls, args): # TODO: this method should also consider config files # TODO: this - cache = CacheConfigS3( - bucket='my-cool-bucket', - path='/', + backend = cache.S3Backend( + bucket='some-bucket', + path='ckuehl/lazy-build/', ) return cls( - context=frozenset(args.context or ()), + context=frozenset(args.context), ignore=frozenset(args.ignore or ()), - cache=cache, + output=frozenset(args.output), + backend=backend, ) - - -CacheConfigS3 = collections.namedtuple('CacheConfigS3', ( - 'bucket', - 'path', -)) diff --git a/lazy_build/context.py b/lazy_build/context.py index 13b9879..cb05219 100644 --- a/lazy_build/context.py +++ b/lazy_build/context.py @@ -9,6 +9,9 @@ import json import os import os.path +import shutil +import tarfile +import tempfile def hash(data): @@ -16,6 +19,7 @@ def hash(data): class BuildContext(collections.namedtuple('BuildContext', ( + 'command', 'files', ))): @@ -23,7 +27,10 @@ class BuildContext(collections.namedtuple('BuildContext', ( @property def hash(self): - return hash(json.dumps(self.files, sort_keys=True).encode('utf8')) + return hash(json.dumps( + (self.command, self.files), + sort_keys=True, + ).encode('utf8')) class FileContext(collections.namedtuple('FileContext', ( @@ -38,9 +45,10 @@ def from_path(cls, path): assert os.path.lexists(path), path if os.path.islink(path): - # TODO: does this properly handle non-utf8 paths? - # (how does Python turn those into strings?) - return cls('link', hash(os.readlink(path).encode('utf8'))) + return cls( + 'link', + hash(os.readlink(path).encode('utf8', 'surrogateescape')), + ) else: with open(path, 'rb') as f: return cls('file', hash(f.read())) @@ -65,7 +73,7 @@ def should_ignore(patterns, path): ) -def build_context(conf): +def build_context(conf, command): ctx = {} fringe = { os.path.relpath(os.path.realpath(path)) for path in conf.context @@ -87,4 +95,30 @@ def build_context(conf): else: ctx[path] = FileContext.from_path(path) - return BuildContext(files=ctx) + return BuildContext(command=command, files=ctx) + + +def package_artifact(conf): + fd, tmp = tempfile.mkstemp() + os.close(fd) + with tarfile.TarFile(tmp, mode='w') as tf: + for output_path in conf.output: + if os.path.isdir(output_path): + for path, _, filenames in os.walk(output_path): + for filename in filenames: + tf.add(os.path.join(path, filename)) + else: + tf.add(output_path) + return tmp + + +def extract_artifact(conf, artifact): + for output_path in conf.output: + if os.path.lexists(output_path): + if os.path.isdir(output_path) and not os.path.islink(output_path): + shutil.rmtree(output_path) + else: + os.remove(output_path) + + with tarfile.TarFile(artifact) as tf: + tf.extractall() diff --git a/setup.py b/setup.py index c31a364..7b85f46 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', ], - install_requires=[], + install_requires=['boto3'], packages=find_packages(exclude=('tests*', 'testing*')), entry_points={ 'console_scripts': ['lazy-build = lazy_build.cli:main'], diff --git a/tests/context_test.py b/tests/context_test.py index 3726411..f49bb67 100644 --- a/tests/context_test.py +++ b/tests/context_test.py @@ -35,7 +35,8 @@ def test_build_context_simple(tmpdir): conf = config.Config( context={tmpdir.strpath, tmpdir.join('a').strpath}, ignore={'d'}, - cache=None, + output=('output',), + backend=None, ) tmpdir.join('a').write(b'foo') tmpdir.join('b').mkdir() @@ -43,8 +44,11 @@ def test_build_context_simple(tmpdir): tmpdir.join('d').mkdir() tmpdir.join('d/e').write(b'baz') tmpdir.join('f').mksymlinkto('/etc/passwd') - assert context.build_context(conf) == context.BuildContext(files={ - 'a': context.FileContext('file', context.hash(b'foo')), - 'b/c': context.FileContext('file', context.hash(b'bar')), - 'f': context.FileContext('link', context.hash(b'/etc/passwd')), - }) + assert context.build_context(conf, 'command') == context.BuildContext( + files={ + 'a': context.FileContext('file', context.hash(b'foo')), + 'b/c': context.FileContext('file', context.hash(b'bar')), + 'f': context.FileContext('link', context.hash(b'/etc/passwd')), + }, + command='command', + )