From 5185668b157eabeae75ab8eb8ececcf69fd0453b Mon Sep 17 00:00:00 2001 From: Zheng Jin Date: Wed, 11 Mar 2020 14:41:43 -0700 Subject: [PATCH 1/3] support specifying a tar file as context Signed-off-by: Zheng Jin --- compose/service.py | 78 +++++++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 29 deletions(-) diff --git a/compose/service.py b/compose/service.py index ebe237b8cf..25a77fafbe 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1083,33 +1083,50 @@ def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_a ) builder = self.client if not cli else _CLIBuilder(progress) - build_output = builder.build( - path=path, - tag=self.image_name, - rm=rm, - forcerm=force_rm, - pull=pull, - nocache=no_cache, - dockerfile=build_opts.get('dockerfile', None), - cache_from=self.get_cache_from(build_opts), - labels=build_opts.get('labels', None), - buildargs=build_args, - network_mode=build_opts.get('network', None), - target=build_opts.get('target', None), - shmsize=parse_bytes(build_opts.get('shm_size')) if build_opts.get('shm_size') else None, - extra_hosts=build_opts.get('extra_hosts', None), - container_limits={ - 'memory': parse_bytes(memory) if memory else None - }, - gzip=gzip, - isolation=build_opts.get('isolation', self.options.get('isolation', None)), - platform=self.platform, - ) + + if os.path.isfile(path): + _path = None + fileobj = open(path, 'rb') + custom_context = True + else: + _path = path + fileobj = None + custom_context = False try: - all_events = list(stream_output(build_output, output_stream)) - except StreamOutputError as e: - raise BuildError(self, six.text_type(e)) + build_output = builder.build( + path=_path, + tag=self.image_name, + rm=rm, + forcerm=force_rm, + pull=pull, + nocache=no_cache, + dockerfile=build_opts.get('dockerfile', None), + cache_from=self.get_cache_from(build_opts), + labels=build_opts.get('labels', None), + buildargs=build_args, + network_mode=build_opts.get('network', None), + target=build_opts.get('target', None), + shmsize=parse_bytes(build_opts.get('shm_size')) if build_opts.get('shm_size') else None, + extra_hosts=build_opts.get('extra_hosts', None), + container_limits={ + 'memory': parse_bytes(memory) if memory else None + }, + gzip=gzip, + isolation=build_opts.get('isolation', self.options.get('isolation', None)), + platform=self.platform, + fileobj=fileobj, + custom_context=custom_context, + ) + + try: + all_events = list(stream_output(build_output, output_stream)) + except StreamOutputError as e: + raise BuildError(self, six.text_type(e)) + + finally: + if fileobj is not None: + fileobj.close() # Ensure the HTTP connection is not reused for another # streaming command, as the Docker daemon can sometimes @@ -1722,7 +1739,7 @@ class _CLIBuilder(object): def __init__(self, progress): self._progress = progress - def build(self, path, tag=None, quiet=False, fileobj=None, + def build(self, path=None, tag=None, quiet=False, fileobj=None, nocache=False, rm=False, timeout=None, custom_context=False, encoding=None, pull=False, forcerm=False, dockerfile=None, container_limits=None, @@ -1783,7 +1800,7 @@ def build(self, path, tag=None, quiet=False, fileobj=None, Returns: A generator for the build output. """ - if dockerfile: + if path and dockerfile: dockerfile = os.path.join(path, dockerfile) iidfile = tempfile.mktemp() @@ -1799,11 +1816,14 @@ def build(self, path, tag=None, quiet=False, fileobj=None, command_builder.add_arg("--tag", tag) command_builder.add_arg("--target", target) command_builder.add_arg("--iidfile", iidfile) - args = command_builder.build([path]) + args = command_builder.build([path or '-']) magic_word = "Successfully built " appear = False - with subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True) as p: + with subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) as p: + if fileobj: + p.stdin.write(fileobj.read().decode()) + p.stdin.close() while True: line = p.stdout.readline() if not line: From edf419c59a8bbb3f1a841c200c1a3d6ae5d5bf0a Mon Sep 17 00:00:00 2001 From: Zheng Jin Date: Thu, 12 Mar 2020 10:33:42 -0700 Subject: [PATCH 2/3] fix flake8 issues Signed-off-by: Zheng Jin --- compose/service.py | 137 +++++++++++++++++++++++++++++---------------- 1 file changed, 88 insertions(+), 49 deletions(-) diff --git a/compose/service.py b/compose/service.py index 25a77fafbe..fa7d035a2c 100644 --- a/compose/service.py +++ b/compose/service.py @@ -10,6 +10,7 @@ import tempfile from collections import namedtuple from collections import OrderedDict +from contextlib import contextmanager from operator import attrgetter import enum @@ -1060,6 +1061,77 @@ def build_spec(secret): return [build_spec(secret) for secret in self.secrets] + @contextmanager + def _prepare_build_args( + self, + no_cache=False, + pull=False, + force_rm=False, + memory=None, + build_args_override=None, + gzip=False, + rm=True, + ): + fileobj = None + + try: + build_opts = self.options.get('build', {}) + + path = rewrite_build_path(build_opts.get('context')) + if os.path.isfile(path): + fileobj = open(path, 'rb') + custom_context = True + path = None + else: + fileobj = None + custom_context = False + + build_args = build_opts.get('args', {}).copy() + if build_args_override: + build_args.update(build_args_override) + for k, v in self._parse_proxy_config().items(): + build_args.setdefault(k, v) + + shmsize = build_opts.get('shm_size') + if shmsize is not None: + shmsize = parse_bytes(shmsize) + + if memory is not None: + memory = parse_bytes(memory) + container_limits = { + 'memory': memory, + } + + isolation = self.options.get('isolation', None) + isolation = build_opts.get('isolation', isolation) + + yield { + 'path': path, + 'fileobj': fileobj, + 'custom_context': custom_context, + 'tag': self.image_name, + 'rm': rm, + 'forcerm': force_rm, + 'pull': pull, + 'nocache': no_cache, + 'dockerfile': build_opts.get('dockerfile', None), + 'cache_from': self.get_cache_from(build_opts), + 'labels': build_opts.get('labels', None), + 'buildargs': build_args, + 'network_mode': build_opts.get('network', None), + 'target': build_opts.get('target', None), + 'shmsize': shmsize, + 'extra_hosts': build_opts.get('extra_hosts', None), + 'container_limits': container_limits, + 'gzip': gzip, + 'isolation': isolation, + 'platform': self.platform, + } + + finally: + if fileobj is not None: + fileobj.close() + def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_args_override=None, gzip=False, rm=True, silent=False, cli=False, progress=None): output_stream = open(os.devnull, 'w') @@ -1067,16 +1139,6 @@ def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_a output_stream = sys.stdout log.info('Building %s' % self.name) - build_opts = self.options.get('build', {}) - - build_args = build_opts.get('args', {}).copy() - if build_args_override: - build_args.update(build_args_override) - - for k, v in self._parse_proxy_config().items(): - build_args.setdefault(k, v) - - path = rewrite_build_path(build_opts.get('context')) if self.platform and version_lt(self.client.api_version, '1.35'): raise OperationFailedError( 'Impossible to perform platform-targeted builds for API version < 1.35' @@ -1084,50 +1146,22 @@ def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_a builder = self.client if not cli else _CLIBuilder(progress) - if os.path.isfile(path): - _path = None - fileobj = open(path, 'rb') - custom_context = True - else: - _path = path - fileobj = None - custom_context = False - - try: - build_output = builder.build( - path=_path, - tag=self.image_name, - rm=rm, - forcerm=force_rm, - pull=pull, - nocache=no_cache, - dockerfile=build_opts.get('dockerfile', None), - cache_from=self.get_cache_from(build_opts), - labels=build_opts.get('labels', None), - buildargs=build_args, - network_mode=build_opts.get('network', None), - target=build_opts.get('target', None), - shmsize=parse_bytes(build_opts.get('shm_size')) if build_opts.get('shm_size') else None, - extra_hosts=build_opts.get('extra_hosts', None), - container_limits={ - 'memory': parse_bytes(memory) if memory else None - }, - gzip=gzip, - isolation=build_opts.get('isolation', self.options.get('isolation', None)), - platform=self.platform, - fileobj=fileobj, - custom_context=custom_context, - ) + with self._prepare_build_args( + no_cache=no_cache, + pull=pull, + force_rm=force_rm, + memory=memory, + build_args_override=build_args_override, + gzip=gzip, + rm=rm, + ) as kwargs: + build_output = builder.build(**kwargs) try: all_events = list(stream_output(build_output, output_stream)) except StreamOutputError as e: raise BuildError(self, six.text_type(e)) - finally: - if fileobj is not None: - fileobj.close() - # Ensure the HTTP connection is not reused for another # streaming command, as the Docker daemon can sometimes # complain about it @@ -1820,7 +1854,12 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, magic_word = "Successfully built " appear = False - with subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) as p: + with subprocess.Popen( + args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=True, + ) as p: if fileobj: p.stdin.write(fileobj.read().decode()) p.stdin.close() From 6dea598d6ddae24b016f654db778e42d3e1aeddd Mon Sep 17 00:00:00 2001 From: Zheng Jin Date: Thu, 12 Mar 2020 10:34:20 -0700 Subject: [PATCH 3/3] fix potential undefined variable bug Signed-off-by: Zheng Jin --- compose/service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/service.py b/compose/service.py index fa7d035a2c..fed19d923a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1169,6 +1169,7 @@ def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_a image_id = None + event = None for event in all_events: if 'stream' in event: match = re.search(r'Successfully built ([0-9a-f]+)', event.get('stream', ''))