From 71964fb357d6d1e7dc891d18e7c819255c92139c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 26 Mar 2018 13:36:45 -0700 Subject: [PATCH 1/3] Support building with Dockerfile outside of context Signed-off-by: Joffrey F --- docker/api/build.py | 10 +++++++++ docker/utils/build.py | 22 +++++++++++++------ docker/utils/utils.py | 12 ++++++++++- tests/integration/api_build_test.py | 33 +++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 7 deletions(-) diff --git a/docker/api/build.py b/docker/api/build.py index 3067c1051a..2a227591fa 100644 --- a/docker/api/build.py +++ b/docker/api/build.py @@ -1,6 +1,7 @@ import json import logging import os +import random from .. import auth from .. import constants @@ -148,6 +149,15 @@ def build(self, path=None, tag=None, quiet=False, fileobj=None, lambda x: x != '' and x[0] != '#', [l.strip() for l in f.read().splitlines()] )) + if dockerfile and os.path.relpath(dockerfile, path).startswith( + '..'): + with open(dockerfile, 'r') as df: + dockerfile = ( + '.dockerfile.{0:x}'.format(random.getrandbits(160)), + df.read() + ) + else: + dockerfile = (dockerfile, None) context = utils.tar( path, exclude=exclude, dockerfile=dockerfile, gzip=gzip ) diff --git a/docker/utils/build.py b/docker/utils/build.py index 894b29936b..0f173476f2 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -2,23 +2,33 @@ import re from ..constants import IS_WINDOWS_PLATFORM +from .utils import create_archive from fnmatch import fnmatch from itertools import chain -from .utils import create_archive + + +_SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/') def tar(path, exclude=None, dockerfile=None, fileobj=None, gzip=False): root = os.path.abspath(path) exclude = exclude or [] + dockerfile = dockerfile or (None, None) + extra_files = [] + if dockerfile[1] is not None: + dockerignore_contents = '\n'.join( + (exclude or ['.dockerignore']) + [dockerfile[0]] + ) + extra_files = [ + ('.dockerignore', dockerignore_contents), + dockerfile, + ] return create_archive( - files=sorted(exclude_paths(root, exclude, dockerfile=dockerfile)), - root=root, fileobj=fileobj, gzip=gzip + files=sorted(exclude_paths(root, exclude, dockerfile=dockerfile[0])), + root=root, fileobj=fileobj, gzip=gzip, extra_files=extra_files ) -_SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/') - - def exclude_paths(root, patterns, dockerfile=None): """ Given a root directory path and a list of .dockerignore patterns, return diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 3cd2be8169..5024e471b2 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -88,13 +88,17 @@ def build_file_list(root): return files -def create_archive(root, files=None, fileobj=None, gzip=False): +def create_archive(root, files=None, fileobj=None, gzip=False, + extra_files=None): if not fileobj: fileobj = tempfile.NamedTemporaryFile() t = tarfile.open(mode='w:gz' if gzip else 'w', fileobj=fileobj) if files is None: files = build_file_list(root) for path in files: + if path in [e[0] for e in extra_files]: + # Extra files override context files with the same name + continue full_path = os.path.join(root, path) i = t.gettarinfo(full_path, arcname=path) @@ -123,6 +127,12 @@ def create_archive(root, files=None, fileobj=None, gzip=False): else: # Directories, FIFOs, symlinks... don't need to be read. t.addfile(i, None) + + for name, contents in extra_files: + info = tarfile.TarInfo(name) + info.size = len(contents) + t.addfile(info, io.BytesIO(contents.encode('utf-8'))) + t.close() fileobj.seek(0) return fileobj diff --git a/tests/integration/api_build_test.py b/tests/integration/api_build_test.py index 13bd8ac576..f411efc490 100644 --- a/tests/integration/api_build_test.py +++ b/tests/integration/api_build_test.py @@ -407,3 +407,36 @@ def test_build_invalid_platform(self): assert excinfo.value.status_code == 400 assert 'invalid platform' in excinfo.exconly() + + def test_build_out_of_context_dockerfile(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + with open(os.path.join(base_dir, 'file.txt'), 'w') as f: + f.write('hello world') + with open(os.path.join(base_dir, '.dockerignore'), 'w') as f: + f.write('.dockerignore\n') + df = tempfile.NamedTemporaryFile() + self.addCleanup(df.close) + df.write(('\n'.join([ + 'FROM busybox', + 'COPY . /src', + 'WORKDIR /src', + ])).encode('utf-8')) + df.flush() + img_name = random_name() + self.tmp_imgs.append(img_name) + stream = self.client.build( + path=base_dir, dockerfile=df.name, tag=img_name, + decode=True + ) + lines = [] + for chunk in stream: + lines.append(chunk) + assert 'Successfully tagged' in lines[-1]['stream'] + + ctnr = self.client.create_container(img_name, 'ls -a') + self.tmp_containers.append(ctnr) + self.client.start(ctnr) + lsdata = self.client.logs(ctnr).strip().split(b'\n') + assert len(lsdata) == 3 + assert sorted([b'.', b'..', b'file.txt']) == sorted(lsdata) From cdbbb06a9500373c473af58b5be47c1ef9ec1ae5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 26 Mar 2018 13:38:13 -0700 Subject: [PATCH 2/3] Move build utils to appropriate file Signed-off-by: Joffrey F --- docker/utils/__init__.py | 6 +-- docker/utils/build.py | 91 +++++++++++++++++++++++++++++++++++++++- docker/utils/utils.py | 89 --------------------------------------- 3 files changed, 93 insertions(+), 93 deletions(-) diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index e70a5e615d..81c8186c84 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -1,13 +1,13 @@ # flake8: noqa -from .build import tar, exclude_paths +from .build import create_archive, exclude_paths, mkbuildcontext, tar from .decorators import check_resource, minimum_version, update_headers from .utils import ( compare_version, convert_port_bindings, convert_volume_binds, - mkbuildcontext, parse_repository_tag, parse_host, + parse_repository_tag, parse_host, kwargs_from_env, convert_filters, datetime_to_timestamp, create_host_config, parse_bytes, parse_env_file, version_lt, version_gte, decode_json_header, split_command, create_ipam_config, create_ipam_pool, parse_devices, normalize_links, convert_service_networks, - format_environment, create_archive, format_extra_hosts + format_environment, format_extra_hosts ) diff --git a/docker/utils/build.py b/docker/utils/build.py index 0f173476f2..783273ee8b 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -1,8 +1,11 @@ +import io import os import re +import six +import tarfile +import tempfile from ..constants import IS_WINDOWS_PLATFORM -from .utils import create_archive from fnmatch import fnmatch from itertools import chain @@ -127,3 +130,89 @@ def match(p): yield f elif matched: yield f + + +def build_file_list(root): + files = [] + for dirname, dirnames, fnames in os.walk(root): + for filename in fnames + dirnames: + longpath = os.path.join(dirname, filename) + files.append( + longpath.replace(root, '', 1).lstrip('/') + ) + + return files + + +def create_archive(root, files=None, fileobj=None, gzip=False, + extra_files=None): + extra_files = extra_files or [] + if not fileobj: + fileobj = tempfile.NamedTemporaryFile() + t = tarfile.open(mode='w:gz' if gzip else 'w', fileobj=fileobj) + if files is None: + files = build_file_list(root) + for path in files: + if path in [e[0] for e in extra_files]: + # Extra files override context files with the same name + continue + full_path = os.path.join(root, path) + + i = t.gettarinfo(full_path, arcname=path) + if i is None: + # This happens when we encounter a socket file. We can safely + # ignore it and proceed. + continue + + # Workaround https://bugs.python.org/issue32713 + if i.mtime < 0 or i.mtime > 8**11 - 1: + i.mtime = int(i.mtime) + + if IS_WINDOWS_PLATFORM: + # Windows doesn't keep track of the execute bit, so we make files + # and directories executable by default. + i.mode = i.mode & 0o755 | 0o111 + + if i.isfile(): + try: + with open(full_path, 'rb') as f: + t.addfile(i, f) + except IOError: + raise IOError( + 'Can not read file in context: {}'.format(full_path) + ) + else: + # Directories, FIFOs, symlinks... don't need to be read. + t.addfile(i, None) + + for name, contents in extra_files: + info = tarfile.TarInfo(name) + info.size = len(contents) + t.addfile(info, io.BytesIO(contents.encode('utf-8'))) + + t.close() + fileobj.seek(0) + return fileobj + + +def mkbuildcontext(dockerfile): + f = tempfile.NamedTemporaryFile() + t = tarfile.open(mode='w', fileobj=f) + if isinstance(dockerfile, io.StringIO): + dfinfo = tarfile.TarInfo('Dockerfile') + if six.PY3: + raise TypeError('Please use io.BytesIO to create in-memory ' + 'Dockerfiles with Python 3') + else: + dfinfo.size = len(dockerfile.getvalue()) + dockerfile.seek(0) + elif isinstance(dockerfile, io.BytesIO): + dfinfo = tarfile.TarInfo('Dockerfile') + dfinfo.size = len(dockerfile.getvalue()) + dockerfile.seek(0) + else: + dfinfo = t.gettarinfo(fileobj=dockerfile, arcname='Dockerfile') + t.addfile(dfinfo, dockerfile) + t.close() + f.seek(0) + return f diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 5024e471b2..fe3b9a5767 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -1,17 +1,13 @@ import base64 -import io import os import os.path import json import shlex -import tarfile -import tempfile from distutils.version import StrictVersion from datetime import datetime import six -from .. import constants from .. import errors from .. import tls @@ -46,29 +42,6 @@ def create_ipam_config(*args, **kwargs): ) -def mkbuildcontext(dockerfile): - f = tempfile.NamedTemporaryFile() - t = tarfile.open(mode='w', fileobj=f) - if isinstance(dockerfile, io.StringIO): - dfinfo = tarfile.TarInfo('Dockerfile') - if six.PY3: - raise TypeError('Please use io.BytesIO to create in-memory ' - 'Dockerfiles with Python 3') - else: - dfinfo.size = len(dockerfile.getvalue()) - dockerfile.seek(0) - elif isinstance(dockerfile, io.BytesIO): - dfinfo = tarfile.TarInfo('Dockerfile') - dfinfo.size = len(dockerfile.getvalue()) - dockerfile.seek(0) - else: - dfinfo = t.gettarinfo(fileobj=dockerfile, arcname='Dockerfile') - t.addfile(dfinfo, dockerfile) - t.close() - f.seek(0) - return f - - def decode_json_header(header): data = base64.b64decode(header) if six.PY3: @@ -76,68 +49,6 @@ def decode_json_header(header): return json.loads(data) -def build_file_list(root): - files = [] - for dirname, dirnames, fnames in os.walk(root): - for filename in fnames + dirnames: - longpath = os.path.join(dirname, filename) - files.append( - longpath.replace(root, '', 1).lstrip('/') - ) - - return files - - -def create_archive(root, files=None, fileobj=None, gzip=False, - extra_files=None): - if not fileobj: - fileobj = tempfile.NamedTemporaryFile() - t = tarfile.open(mode='w:gz' if gzip else 'w', fileobj=fileobj) - if files is None: - files = build_file_list(root) - for path in files: - if path in [e[0] for e in extra_files]: - # Extra files override context files with the same name - continue - full_path = os.path.join(root, path) - - i = t.gettarinfo(full_path, arcname=path) - if i is None: - # This happens when we encounter a socket file. We can safely - # ignore it and proceed. - continue - - # Workaround https://bugs.python.org/issue32713 - if i.mtime < 0 or i.mtime > 8**11 - 1: - i.mtime = int(i.mtime) - - if constants.IS_WINDOWS_PLATFORM: - # Windows doesn't keep track of the execute bit, so we make files - # and directories executable by default. - i.mode = i.mode & 0o755 | 0o111 - - if i.isfile(): - try: - with open(full_path, 'rb') as f: - t.addfile(i, f) - except IOError: - raise IOError( - 'Can not read file in context: {}'.format(full_path) - ) - else: - # Directories, FIFOs, symlinks... don't need to be read. - t.addfile(i, None) - - for name, contents in extra_files: - info = tarfile.TarInfo(name) - info.size = len(contents) - t.addfile(info, io.BytesIO(contents.encode('utf-8'))) - - t.close() - fileobj.seek(0) - return fileobj - - def compare_version(v1, v2): """Compare docker versions From cb99cc6915b81c4b5959ccfabd8e671463a9c179 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 27 Mar 2018 10:22:17 -0700 Subject: [PATCH 3/3] Improve extra_files override check Signed-off-by: Joffrey F --- docker/utils/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/utils/build.py b/docker/utils/build.py index 783273ee8b..b644c9fca1 100644 --- a/docker/utils/build.py +++ b/docker/utils/build.py @@ -152,8 +152,9 @@ def create_archive(root, files=None, fileobj=None, gzip=False, t = tarfile.open(mode='w:gz' if gzip else 'w', fileobj=fileobj) if files is None: files = build_file_list(root) + extra_names = set(e[0] for e in extra_files) for path in files: - if path in [e[0] for e in extra_files]: + if path in extra_names: # Extra files override context files with the same name continue full_path = os.path.join(root, path)