Skip to content

Commit

Permalink
Read/use Directory Transport version from Skopeo
Browse files Browse the repository at this point in the history
Adds comparators for a new class Version which represents a skopeo
directory transport version. This will allow us to easily support
future changes.

containers/image#419
fixes: #3703
  • Loading branch information
asmacdo committed Jun 12, 2018
1 parent 3a6d9a9 commit 1b1caac
Show file tree
Hide file tree
Showing 4 changed files with 280 additions and 10 deletions.
117 changes: 117 additions & 0 deletions common/pulp_docker/common/transport.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
class Version:
"""
Represents a version of a Skopeo Directory Transport version file.
This class enables rich comparisons between versions. All comparisons are implemented because
functools.total_ordering is not available in Python 2.6.
"""
def __init__(self, version):
"""
Args:
version (str): version number
"""
self.version = version

@classmethod
def from_file(cls, path):
"""
Creates a Version from a Skopeo version file.
Args:
path (str): path to the version file
Return:
(Version): instance
Raises:
(IOError): when path does not exist
"""
with open(path) as version_file:
version = version_file.readline().split(':')[1].strip()
return cls(version)

@property
def components(self):
"""
Return:
(tuple) of 2 integer components of the release, (X, Y)
"""
components = [int(comp) for comp in self.version.split(".")]
assert len(components) == 2, "Directory Transport Versions must be in the form X.Y"
return tuple(components)

def __eq__(self, other):
"""
Args:
other (Version): version to compare with
Return:
(bool) True if self == other
"""
return self.version == other.version

def __ne__(self, other):
"""
Args:
other (Version): version to compare with
Return:
(bool) True if self != other
"""
return not self.version == other.version

def __gt__(self, other):
"""
Args:
other (Version): version to compare with
Return:
(bool) True if self > other
"""
comparisons = zip(self.components, other.components)
for s, o in comparisons:
if s != o:
return s > o
return False

def __ge__(self, other):
"""
Args:
other (Version): version to compare with
Return:
(bool) True if self >= other
"""
op_result = self.__gt__(other)
return op_result or self == other

def __lt__(self, other):
"""
Args:
other (Version): version to compare with
Return:
(bool) True if self < other
"""
comparisons = zip(self.components, other.components)
for s, o in comparisons:
if s != o:
return not s > o

def __le__(self, other):
"""
Args:
other (Version): version to compare with
Return:
(bool) True if self <= other
"""
op_result = self.__lt__(other)
return op_result or self == other

def __repr__(self):
"""
Return:
(string) in the format that skopeo uses.
"""
return "Directory Transport Version: {v}\n".format(v=self.version)
80 changes: 80 additions & 0 deletions common/test/unit/test_transport.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import mock
import unittest

from pulp_docker.common import transport


class TestVersion(unittest.TestCase):

def setUp(self):
self.v00 = transport.Version("0.0")
self.v10 = transport.Version("1.0")
self.v11 = transport.Version("1.1")
self.v199 = transport.Version("1.99")
self.v20 = transport.Version("2.0")
self.copy_v11 = transport.Version("1.1")
self.copy_v00 = transport.Version("0.0")

def test_read(self):
"""
Ensure that the file line is correctly parsed.
"""
mock_version = "Directory Transport Version: 1.1\n"
with mock.patch('pulp_docker.common.transport.open', create=True) as mopen:
mock_readline = mopen.return_value.__enter__.return_value.readline
mock_readline.return_value = mock_version
version = transport.Version.from_file("path-to-version")

self.assertEqual(version.version, '1.1')

def test_read_nonexistant(self):
try:
with mock.patch('pulp_docker.common.transport.open', create=True) as mopen:
mopen.side_effect = IOError
transport.Version.from_file("path-to-version")
except IOError:
pass
else:
raise AssertionError("IOError from open should bubble up.")

def test_eq(self):
self.assertTrue(self.v11 == self.copy_v11)
self.assertTrue(self.v00 == self.copy_v00)

self.assertFalse(self.v10 == self.v11)

def test_inequality(self):
self.assertTrue(self.v10 != self.v11)

self.assertFalse(self.v00 != self.copy_v00)
self.assertFalse(self.v11 != self.copy_v11)

def test_gt(self):
self.assertTrue(self.v11 > self.v10)

self.assertFalse(self.v10 > self.v11)
self.assertFalse(self.v11 > self.copy_v11)

def test_gt_for_large_y_release(self):
self.assertTrue(self.v20 > self.v199)

def test_ge(self):
self.assertTrue(self.v10 >= self.v10)
self.assertTrue(self.v11 >= self.v10)

self.assertFalse(self.v10 >= self.v11)

def test_lt(self):
self.assertTrue(self.v10 < self.v11)

self.assertFalse(self.v11 < self.v00)
self.assertFalse(self.v11 < self.copy_v11)

def test_lt_for_large_y_release(self):
self.assertTrue(self.v199 < self.v20)

def test_le(self):
self.assertTrue(self.v10 <= self.v10)
self.assertTrue(self.v10 <= self.v11)

self.assertFalse(self.v11 <= self.v10)
11 changes: 9 additions & 2 deletions plugins/pulp_docker/plugins/importers/upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
from pulp.plugins.util.publish_step import PluginStep, GetLocalUnitsStep
from pulp.server.db import model as pulp_models

from pulp_docker.common import constants, error_codes, tarutils
from pulp_docker.common import constants, error_codes, tarutils, transport
from pulp_docker.plugins import models
from pulp_docker.plugins.importers import v1_sync
from pulp.plugins.util import verification
Expand Down Expand Up @@ -409,7 +409,14 @@ def process_main(self, item=None):
if not checksum_type:
# Never assume. But oh well
checksum_type = "sha256"
blob_src_path = os.path.join(self.get_working_dir(), checksum + '.tar')

blob_src_path = os.path.join(self.get_working_dir(), checksum)
version_file_path = os.path.join(self.get_working_dir(), 'version')
transport_version = transport.Version.from_file(version_file_path)
if transport_version < transport.Version('1.1'):
# Directory Transport Version 1.0 expects each file to end in .tar
blob_src_path = "{path}.tar".format(path=blob_src_path)

try:
fobj = open(blob_src_path)
except IOError:
Expand Down
82 changes: 74 additions & 8 deletions plugins/test/unit/plugins/importers/test_upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@ def setUp(self):
self.work_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.work_dir, ignore_errors=True)

def test_AddUnits(self, _repo_controller, _Manifest_save, _Blob_save):
@mock.patch('pulp_docker.plugins.importers.upload.transport.Version.from_file')
def test_AddUnits(self, mock_v, _repo_controller, _Manifest_save, _Blob_save):
mock_v.return_value = upload.transport.Version('1.0')
# This is where we will untar the image
step_work_dir = os.path.join(self.work_dir, "working_dir")
os.makedirs(step_work_dir)
Expand Down Expand Up @@ -73,6 +75,58 @@ def test_AddUnits(self, _repo_controller, _Manifest_save, _Blob_save):
units[0],
parent.uploaded_unit)

@mock.patch('pulp_docker.plugins.importers.upload.transport.Version.from_file')
def test_AddUnits_skopeo11(self, mock_v, _repo_controller, _Manifest_save, _Blob_save):
mock_v.return_value = upload.transport.Version('1.1')

# This is where we will untar the image
step_work_dir = os.path.join(self.work_dir, "working_dir")
os.makedirs(step_work_dir)

img, layers = self._create_image(transport_version="1.1")
manifest_data = dict(layers=[dict(digest=x['digest'],
mediaType="ignored")
for x in layers],
config=dict(digest="abc"),
schemaVersion=2)
units = [
models.Manifest.from_json(json.dumps(manifest_data), digest="012"),
]
units.extend(models.Blob(digest="sha256:%s" % x['digest'])
for x in layers)

parent = mock.MagicMock(file_path=img, parent=None, uploaded_unit=None)
parent.v2_step_get_local_units.units_to_download = units
step = upload.AddUnits(step_type=constants.UPLOAD_STEP_SAVE,
working_dir=step_work_dir)
step.parent = parent
step.process_lifecycle()

dst_blobs = []

# Make sure the blobs were created, and not compressed
for i, layer in enumerate(layers):
dst = os.path.join(step_work_dir, "sha256:%s" % layer['digest'])
with open(dst) as content:
self.assertEquals(layer['content'], content.read())
dst_blobs.append(dst)

# Make sure we called save_and_import_content
self.assertEquals(
[mock.call(x) for x in dst_blobs],
_Blob_save.call_args_list)
_Manifest_save.assert_called_once_with(
os.path.join(step_work_dir, "012"))

# Make sure associate_single_unit got called
repo_obj = parent.get_repo.return_value.repo_obj
self.assertEquals(
[mock.call(repo_obj, x) for x in units],
_repo_controller.associate_single_unit.call_args_list)
self.assertEquals(
units[0],
parent.uploaded_unit)

@mock.patch("pulp_docker.plugins.models.Manifest.objects")
def test_AddUnits__mf_exists(self, _Manifest_objects, _repo_controller,
_Manifest_save, _Blob_save):
Expand Down Expand Up @@ -106,7 +160,10 @@ def test_AddUnits__mf_exists(self, _Manifest_objects, _repo_controller,
_Manifest_objects.get.return_value,
parent.uploaded_unit)

def test_AddUnits_error_bad_checksum(self, _repo_controller, _Manifest_save, _Blob_save):
@mock.patch('pulp_docker.plugins.importers.upload.transport.Version.from_file')
def test_AddUnits_error_bad_checksum(self, mock_v, _repo_controller, _Manifest_save,
_Blob_save):
mock_v.return_value = upload.transport.Version('1.0')
# This is where we will untar the image
step_work_dir = os.path.join(self.work_dir, "working_dir")
os.makedirs(step_work_dir)
Expand Down Expand Up @@ -136,8 +193,11 @@ def test_AddUnits_error_bad_checksum(self, _repo_controller, _Manifest_save, _Bl
"Checksum bad-digest (sha256) does not validate",
str(ctx.exception))

def test_AddUnits_error_missing_layer(self, _repo_controller, _Manifest_save, _Blob_save):
@mock.patch('pulp_docker.plugins.importers.upload.transport.Version.from_file')
def test_AddUnits_error_missing_layer(self, mock_v, _repo_controller, _Manifest_save,
_Blob_save):
# This is where we will untar the image
mock_v.return_value = upload.transport.Version('1.0')
step_work_dir = os.path.join(self.work_dir, "working_dir")
os.makedirs(step_work_dir)

Expand Down Expand Up @@ -168,10 +228,12 @@ def test_AddUnits_error_missing_layer(self, _repo_controller, _Manifest_save, _B
"Layer this-is-missing.tar is not present in the image",
str(ctx.exception))

@mock.patch('pulp_docker.plugins.importers.upload.transport.Version.from_file')
@mock.patch("pulp_docker.plugins.importers.upload.models.Manifest.objects")
@mock.patch("pulp_docker.plugins.importers.upload.models.Tag.objects")
def test_AddTags(self, _Tag_objects, _Manifest_objects, _repos,
_Manifest_save, _Blob_save):
def test_AddTags(self, _Tag_objects, _Manifest_objects, mock_v, _repos, _Manifest_save,
_Blob_save):
mock_v.return_value = upload.transport.Version('1.0')
_Manifest_objects.filter.return_value.count.return_value = 1
_Manifest_objects.filter.return_value.__getitem__.return_value.schema_version = 42

Expand All @@ -191,7 +253,8 @@ def test_AddTags(self, _Tag_objects, _Manifest_objects, _repos,
schema_version=42, manifest_digest='sha256:123',
pulp_user_metadata=None)

def test_AddTags__error_no_name(self, _repo_controller, _Manifest_save, _Blob_save):
@mock.patch('pulp_docker.plugins.importers.upload.transport.Version.from_file')
def test_AddTags__error_no_name(self, mock_v, _repo_controller, _Manifest_save, _Blob_save):
# This is where we will untar the image
step_work_dir = os.path.join(self.work_dir, "working_dir")
os.makedirs(step_work_dir)
Expand Down Expand Up @@ -229,7 +292,7 @@ def _create_layer(self, content):
fobj.seek(0)
return fobj, sha.hexdigest()

def _create_image(self, with_bad_checksum=False):
def _create_image(self, with_bad_checksum=False, transport_version="1.0"):
fname = os.path.join(self.work_dir, "image.tar")
tobj = tarfile.TarFile(fname, mode="w")
layers = []
Expand All @@ -238,7 +301,10 @@ def _create_image(self, with_bad_checksum=False):
fobj, digest = self._create_layer(content=content)
if with_bad_checksum and i == 1:
digest = "bad-digest"
tinfo = tobj.gettarinfo(arcname="%s.tar" % digest, fileobj=fobj)
if transport_version == "1.0":
tinfo = tobj.gettarinfo(arcname="%s.tar" % digest, fileobj=fobj)
else:
tinfo = tobj.gettarinfo(arcname=digest, fileobj=fobj)
tinfo.uid = tinfo.gid = 0
tinfo.uname = tinfo.gname = "root"
tobj.addfile(tinfo, fileobj=fobj)
Expand Down

0 comments on commit 1b1caac

Please sign in to comment.