diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000000..b73a78eb95 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,88 @@ +#!groovy + +def imageNameBase = "dockerbuildbot/docker-py" +def imageNamePy2 +def imageNamePy3 +def images = [:] +def dockerVersions = ["1.12.0", "1.13.0-rc3"] + +def buildImage = { name, buildargs, pyTag -> + img = docker.image(name) + try { + img.pull() + } catch (Exception exc) { + img = docker.build(name, buildargs) + img.push() + } + images[pyTag] = img.id +} + +def buildImages = { -> + wrappedNode(label: "ubuntu && !zfs && amd64", cleanWorkspace: true) { + stage("build image") { + checkout(scm) + + imageNamePy2 = "${imageNameBase}:py2-${gitCommit()}" + imageNamePy3 = "${imageNameBase}:py3-${gitCommit()}" + + buildImage(imageNamePy2, ".", "py2.7") + buildImage(imageNamePy3, "-f Dockerfile-py3 .", "py3.5") + } + } +} + +def runTests = { Map settings -> + def dockerVersion = settings.get("dockerVersion", null) + def pythonVersion = settings.get("pythonVersion", null) + def testImage = settings.get("testImage", null) + + if (!testImage) { + throw new Exception("Need test image object, e.g.: `runTests(testImage: img)`") + } + if (!dockerVersion) { + throw new Exception("Need Docker version to test, e.g.: `runTests(dockerVersion: '1.12.3')`") + } + if (!pythonVersion) { + throw new Exception("Need Python version being tested, e.g.: `runTests(pythonVersion: 'py2.7')`") + } + + { -> + wrappedNode(label: "ubuntu && !zfs && amd64", cleanWorkspace: true) { + stage("test python=${pythonVersion} / docker=${dockerVersion}") { + checkout(scm) + def dindContainerName = "dpy-dind-\$BUILD_NUMBER-\$EXECUTOR_NUMBER" + def testContainerName = "dpy-tests-\$BUILD_NUMBER-\$EXECUTOR_NUMBER" + try { + sh """docker run -d --name ${dindContainerName} -v /tmp --privileged \\ + dockerswarm/dind:${dockerVersion} docker daemon -H tcp://0.0.0.0:2375 + """ + sh """docker run \\ + --name ${testContainerName} --volumes-from ${dindContainerName} \\ + -e 'DOCKER_HOST=tcp://docker:2375' \\ + --link=${dindContainerName}:docker \\ + ${testImage} \\ + py.test -v -rxs tests/integration + """ + } finally { + sh """ + docker stop ${dindContainerName} ${testContainerName} + docker rm -vf ${dindContainerName} ${testContainerName} + """ + } + } + } + } +} + + +buildImages() + +def testMatrix = [failFast: false] + +for (imgKey in new ArrayList(images.keySet())) { + for (version in dockerVersions) { + testMatrix["${imgKey}_${version}"] = runTests([testImage: images[imgKey], dockerVersion: version, pythonVersion: imgKey]) + } +} + +parallel(testMatrix) diff --git a/docker/errors.py b/docker/errors.py index 05f4cae5c1..95c462b9d2 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -93,6 +93,10 @@ class InvalidConfigFile(DockerException): pass +class InvalidArgument(DockerException): + pass + + class DeprecatedMethod(DockerException): pass diff --git a/docker/types/services.py b/docker/types/services.py index 5041f89d92..93503dc054 100644 --- a/docker/types/services.py +++ b/docker/types/services.py @@ -1,6 +1,7 @@ import six from .. import errors +from ..constants import IS_WINDOWS_PLATFORM from ..utils import format_environment, split_command @@ -131,10 +132,11 @@ def __init__(self, target, source, type='volume', read_only=False, self['Target'] = target self['Source'] = source if type not in ('bind', 'volume'): - raise errors.DockerError( + raise errors.InvalidArgument( 'Only acceptable mount types are `bind` and `volume`.' ) self['Type'] = type + self['ReadOnly'] = read_only if type == 'bind': if propagation is not None: @@ -142,7 +144,7 @@ def __init__(self, target, source, type='volume', read_only=False, 'Propagation': propagation } if any([labels, driver_config, no_copy]): - raise errors.DockerError( + raise errors.InvalidArgument( 'Mount type is binding but volume options have been ' 'provided.' ) @@ -157,7 +159,7 @@ def __init__(self, target, source, type='volume', read_only=False, if volume_opts: self['VolumeOptions'] = volume_opts if propagation: - raise errors.DockerError( + raise errors.InvalidArgument( 'Mount type is volume but `propagation` argument has been ' 'provided.' ) @@ -166,16 +168,25 @@ def __init__(self, target, source, type='volume', read_only=False, def parse_mount_string(cls, string): parts = string.split(':') if len(parts) > 3: - raise errors.DockerError( + raise errors.InvalidArgument( 'Invalid mount format "{0}"'.format(string) ) if len(parts) == 1: - return cls(target=parts[0]) + return cls(target=parts[0], source=None) else: target = parts[1] source = parts[0] - read_only = not (len(parts) == 3 or parts[2] == 'ro') - return cls(target, source, read_only=read_only) + mount_type = 'volume' + if source.startswith('/') or ( + IS_WINDOWS_PLATFORM and source[0].isalpha() and + source[1] == ':' + ): + # FIXME: That windows condition will fail earlier since we + # split on ':'. We should look into doing a smarter split + # if we detect we are on Windows. + mount_type = 'bind' + read_only = not (len(parts) == 2 or parts[2] == 'rw') + return cls(target, source, read_only=read_only, type=mount_type) class Resources(dict): @@ -228,7 +239,7 @@ def __init__(self, parallelism=0, delay=None, failure_action='continue'): if delay is not None: self['Delay'] = delay if failure_action not in ('pause', 'continue'): - raise errors.DockerError( + raise errors.InvalidArgument( 'failure_action must be either `pause` or `continue`.' ) self['FailureAction'] = failure_action diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 4e5f454906..e12fcf00dc 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -175,11 +175,17 @@ def should_check_directory(directory_path, exclude_patterns, include_patterns): # docker logic (2016-10-27): # https://github.com/docker/docker/blob/bc52939b0455116ab8e0da67869ec81c1a1c3e2c/pkg/archive/archive.go#L640-L671 - path_with_slash = directory_path + os.sep - possible_child_patterns = [pattern for pattern in include_patterns if - (pattern + os.sep).startswith(path_with_slash)] - directory_included = should_include(directory_path, exclude_patterns, - include_patterns) + def normalize_path(path): + return path.replace(os.path.sep, '/') + + path_with_slash = normalize_path(directory_path) + '/' + possible_child_patterns = [ + pattern for pattern in map(normalize_path, include_patterns) + if (pattern + '/').startswith(path_with_slash) + ] + directory_included = should_include( + directory_path, exclude_patterns, include_patterns + ) return directory_included or len(possible_child_patterns) > 0 @@ -195,9 +201,11 @@ def get_paths(root, exclude_patterns, include_patterns, has_exceptions=False): # by mutating the dirs we're iterating over. # This looks strange, but is considered the correct way to skip # traversal. See https://docs.python.org/2/library/os.html#os.walk - dirs[:] = [d for d in dirs if - should_check_directory(os.path.join(parent, d), - exclude_patterns, include_patterns)] + dirs[:] = [ + d for d in dirs if should_check_directory( + os.path.join(parent, d), exclude_patterns, include_patterns + ) + ] for path in dirs: if should_include(os.path.join(parent, path), @@ -213,7 +221,7 @@ def get_paths(root, exclude_patterns, include_patterns, has_exceptions=False): def match_path(path, pattern): - pattern = pattern.rstrip('/') + pattern = pattern.rstrip('/' + os.path.sep) if pattern: pattern = os.path.relpath(pattern) diff --git a/docker/version.py b/docker/version.py index dd7995059d..6979eeca46 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "2.0.0" +version = "2.0.1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) diff --git a/docs/change-log.md b/docs/change-log.md index b5998d47c3..91eafcc4dc 100644 --- a/docs/change-log.md +++ b/docs/change-log.md @@ -1,6 +1,20 @@ Change log ========== +2.0.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/28?closed=1) + +### Bugfixes + +* Fixed a bug where forward slashes in some .dockerignore patterns weren't + being parsed correctly on Windows +* Fixed a bug where `Mount.parse_mount_string` would never set the read_only + parameter on the resulting `Mount`. +* Fixed a bug where `Mount.parse_mount_string` would incorrectly mark host + binds as being of `volume` type. + 2.0.0 ----- diff --git a/tests/helpers.py b/tests/helpers.py index 53cf57ad70..1e42363144 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -74,3 +74,7 @@ def force_leave_swarm(client): continue else: return + + +def swarm_listen_addr(): + return '0.0.0.0:{0}'.format(random.randrange(10000, 25000)) diff --git a/tests/integration/api_service_test.py b/tests/integration/api_service_test.py index 04f2fe0bcb..fc7940023f 100644 --- a/tests/integration/api_service_test.py +++ b/tests/integration/api_service_test.py @@ -10,7 +10,7 @@ class ServiceTest(BaseAPIIntegrationTest): def setUp(self): super(ServiceTest, self).setUp() self.client.leave_swarm(force=True) - self.client.init_swarm('eth0') + self.init_swarm() def tearDown(self): super(ServiceTest, self).tearDown() diff --git a/tests/integration/api_swarm_test.py b/tests/integration/api_swarm_test.py index a10437b1e3..a8f439c8b5 100644 --- a/tests/integration/api_swarm_test.py +++ b/tests/integration/api_swarm_test.py @@ -17,39 +17,37 @@ def tearDown(self): @requires_api_version('1.24') def test_init_swarm_simple(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() @requires_api_version('1.24') def test_init_swarm_force_new_cluster(self): pytest.skip('Test stalls the engine on 1.12.0') - assert self.client.init_swarm('eth0') + assert self.init_swarm() version_1 = self.client.inspect_swarm()['Version']['Index'] - assert self.client.init_swarm('eth0', force_new_cluster=True) + assert self.client.init_swarm(force_new_cluster=True) version_2 = self.client.inspect_swarm()['Version']['Index'] assert version_2 != version_1 @requires_api_version('1.24') def test_init_already_in_cluster(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() with pytest.raises(docker.errors.APIError): - self.client.init_swarm('eth0') + self.init_swarm() @requires_api_version('1.24') def test_init_swarm_custom_raft_spec(self): spec = self.client.create_swarm_spec( snapshot_interval=5000, log_entries_for_slow_followers=1200 ) - assert self.client.init_swarm( - advertise_addr='eth0', swarm_spec=spec - ) + assert self.init_swarm(swarm_spec=spec) swarm_info = self.client.inspect_swarm() assert swarm_info['Spec']['Raft']['SnapshotInterval'] == 5000 assert swarm_info['Spec']['Raft']['LogEntriesForSlowFollowers'] == 1200 @requires_api_version('1.24') def test_leave_swarm(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() with pytest.raises(docker.errors.APIError) as exc_info: self.client.leave_swarm() exc_info.value.response.status_code == 500 @@ -61,7 +59,7 @@ def test_leave_swarm(self): @requires_api_version('1.24') def test_update_swarm(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() swarm_info_1 = self.client.inspect_swarm() spec = self.client.create_swarm_spec( snapshot_interval=5000, log_entries_for_slow_followers=1200, @@ -92,7 +90,7 @@ def test_update_swarm(self): @requires_api_version('1.24') def test_update_swarm_name(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() swarm_info_1 = self.client.inspect_swarm() spec = self.client.create_swarm_spec( node_cert_expiry=7776000000000000, name='reimuhakurei' @@ -110,7 +108,7 @@ def test_update_swarm_name(self): @requires_api_version('1.24') def test_list_nodes(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() nodes_list = self.client.nodes() assert len(nodes_list) == 1 node = nodes_list[0] @@ -129,7 +127,7 @@ def test_list_nodes(self): @requires_api_version('1.24') def test_inspect_node(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() nodes_list = self.client.nodes() assert len(nodes_list) == 1 node = nodes_list[0] @@ -139,7 +137,7 @@ def test_inspect_node(self): @requires_api_version('1.24') def test_update_node(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() nodes_list = self.client.nodes() node = nodes_list[0] orig_spec = node['Spec'] @@ -162,7 +160,7 @@ def test_update_node(self): @requires_api_version('1.24') def test_remove_main_node(self): - assert self.client.init_swarm('eth0') + assert self.init_swarm() nodes_list = self.client.nodes() node_id = nodes_list[0]['ID'] with pytest.raises(docker.errors.NotFound): diff --git a/tests/integration/base.py b/tests/integration/base.py index ea43d056e5..4a41e6b81a 100644 --- a/tests/integration/base.py +++ b/tests/integration/base.py @@ -5,6 +5,7 @@ from docker.utils import kwargs_from_env import six +from .. import helpers BUSYBOX = 'busybox:buildroot-2014.02' @@ -90,3 +91,8 @@ def execute(self, container, cmd, exit_code=0, **kwargs): msg = "Expected `{}` to exit with code {} but returned {}:\n{}".format( " ".join(cmd), exit_code, actual_exit_code, output) assert actual_exit_code == exit_code, msg + + def init_swarm(self, **kwargs): + return self.client.init_swarm( + 'eth0', listen_addr=helpers.swarm_listen_addr(), **kwargs + ) diff --git a/tests/integration/models_nodes_test.py b/tests/integration/models_nodes_test.py index 0199d69303..9fd16593ac 100644 --- a/tests/integration/models_nodes_test.py +++ b/tests/integration/models_nodes_test.py @@ -1,5 +1,7 @@ import unittest + import docker + from .. import helpers @@ -12,7 +14,7 @@ def tearDown(self): def test_list_get_update(self): client = docker.from_env() - client.swarm.init() + client.swarm.init(listen_addr=helpers.swarm_listen_addr()) nodes = client.nodes.list() assert len(nodes) == 1 assert nodes[0].attrs['Spec']['Role'] == 'manager' diff --git a/tests/integration/models_services_test.py b/tests/integration/models_services_test.py index baa40a9120..a795df9841 100644 --- a/tests/integration/models_services_test.py +++ b/tests/integration/models_services_test.py @@ -11,7 +11,7 @@ class ServiceTest(unittest.TestCase): def setUpClass(cls): client = docker.from_env() helpers.force_leave_swarm(client) - client.swarm.init() + client.swarm.init(listen_addr=helpers.swarm_listen_addr()) @classmethod def tearDownClass(cls): diff --git a/tests/integration/models_swarm_test.py b/tests/integration/models_swarm_test.py index 72bf9e5c92..4f177f1005 100644 --- a/tests/integration/models_swarm_test.py +++ b/tests/integration/models_swarm_test.py @@ -1,5 +1,7 @@ import unittest + import docker + from .. import helpers @@ -12,7 +14,9 @@ def tearDown(self): def test_init_update_leave(self): client = docker.from_env() - client.swarm.init(snapshot_interval=5000) + client.swarm.init( + snapshot_interval=5000, listen_addr=helpers.swarm_listen_addr() + ) assert client.swarm.attrs['Spec']['Raft']['SnapshotInterval'] == 5000 client.swarm.update(snapshot_interval=10000) assert client.swarm.attrs['Spec']['Raft']['SnapshotInterval'] == 10000 diff --git a/tests/unit/dockertypes_test.py b/tests/unit/dockertypes_test.py index 2480b9ef92..d11e4f03f0 100644 --- a/tests/unit/dockertypes_test.py +++ b/tests/unit/dockertypes_test.py @@ -5,11 +5,16 @@ import pytest from docker.constants import DEFAULT_DOCKER_API_VERSION -from docker.errors import InvalidVersion +from docker.errors import InvalidArgument, InvalidVersion from docker.types import ( - EndpointConfig, HostConfig, IPAMConfig, IPAMPool, LogConfig, Ulimit, + EndpointConfig, HostConfig, IPAMConfig, IPAMPool, LogConfig, Mount, Ulimit, ) +try: + from unittest import mock +except: + import mock + def create_host_config(*args, **kwargs): return HostConfig(*args, **kwargs) @@ -253,3 +258,53 @@ def test_create_ipam_config(self): 'IPRange': None, }] }) + + +class TestMounts(unittest.TestCase): + def test_parse_mount_string_ro(self): + mount = Mount.parse_mount_string("/foo/bar:/baz:ro") + assert mount['Source'] == "/foo/bar" + assert mount['Target'] == "/baz" + assert mount['ReadOnly'] is True + + def test_parse_mount_string_rw(self): + mount = Mount.parse_mount_string("/foo/bar:/baz:rw") + assert mount['Source'] == "/foo/bar" + assert mount['Target'] == "/baz" + assert not mount['ReadOnly'] + + def test_parse_mount_string_short_form(self): + mount = Mount.parse_mount_string("/foo/bar:/baz") + assert mount['Source'] == "/foo/bar" + assert mount['Target'] == "/baz" + assert not mount['ReadOnly'] + + def test_parse_mount_string_no_source(self): + mount = Mount.parse_mount_string("foo/bar") + assert mount['Source'] is None + assert mount['Target'] == "foo/bar" + assert not mount['ReadOnly'] + + def test_parse_mount_string_invalid(self): + with pytest.raises(InvalidArgument): + Mount.parse_mount_string("foo:bar:baz:rw") + + def test_parse_mount_named_volume(self): + mount = Mount.parse_mount_string("foobar:/baz") + assert mount['Source'] == 'foobar' + assert mount['Target'] == '/baz' + assert mount['Type'] == 'volume' + + def test_parse_mount_bind(self): + mount = Mount.parse_mount_string('/foo/bar:/baz') + assert mount['Source'] == "/foo/bar" + assert mount['Target'] == "/baz" + assert mount['Type'] == 'bind' + + @pytest.mark.xfail + def test_parse_mount_bind_windows(self): + with mock.patch('docker.types.services.IS_WINDOWS_PLATFORM', True): + mount = Mount.parse_mount_string('C:/foo/bar:/baz') + assert mount['Source'] == "C:/foo/bar" + assert mount['Target'] == "/baz" + assert mount['Type'] == 'bind' diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 743d076da3..cf00616d3d 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -780,6 +780,16 @@ def test_directory_with_subdir_exception(self): ]) ) + @pytest.mark.skipif( + not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows' + ) + def test_directory_with_subdir_exception_win32_pathsep(self): + assert self.exclude(['foo', '!foo\\bar']) == convert_paths( + self.all_paths - set([ + 'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3' + ]) + ) + def test_directory_with_wildcard_exception(self): assert self.exclude(['foo', '!foo/*.py']) == convert_paths( self.all_paths - set([ @@ -792,6 +802,14 @@ def test_subdirectory(self): self.all_paths - set(['foo/bar', 'foo/bar/a.py']) ) + @pytest.mark.skipif( + not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows' + ) + def test_subdirectory_win32_pathsep(self): + assert self.exclude(['foo\\bar']) == convert_paths( + self.all_paths - set(['foo/bar', 'foo/bar/a.py']) + ) + class TarTest(unittest.TestCase): def test_tar_with_excludes(self):