From 6a148528bea91ee965adf2a8ec57981fbf423244 Mon Sep 17 00:00:00 2001 From: andamian Date: Wed, 29 Nov 2017 15:57:11 -0800 Subject: [PATCH] Copy not using md5 before transfer (#100) * Fixed md5 check before copy file from vospace --- .travis.yml | 1 - vofs/setup.cfg | 4 ++-- vofs/setup.py | 1 - vofs/tox.ini | 2 +- vos/setup.cfg | 2 +- vos/setup.py | 1 - vos/tox.ini | 2 +- vos/vos/tests/test_commonparser.py | 8 ++++---- vos/vos/tests/test_vos.py | 28 ++++++++++++++++++++++++++++ vos/vos/vos.py | 18 ++++++++++++++++-- 10 files changed, 53 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 825d2babb..c6ca9d930 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: python python: - 2.7 - - 3.3 - 3.4 - 3.5 - 3.6 diff --git a/vofs/setup.cfg b/vofs/setup.cfg index 5ce5e6211..c1ae39303 100644 --- a/vofs/setup.cfg +++ b/vofs/setup.cfg @@ -49,9 +49,9 @@ license = AGPLv3 url = http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/vospace edit_on_github = False github_project = opencadc/vostools -install_requires = vos>=3.0 BitVector>=3.4.4,<4.0 fusepy>=2.0.4 +install_requires = vos>=3.0.3 BitVector>=3.4.4,<4.0 fusepy>=2.0.4 # version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440) -version = 3.0.1 +version = 3.0.2 [entry_points] #astropy-package-template-example = packagename.example_mod:main diff --git a/vofs/setup.py b/vofs/setup.py index f3a271b97..86cc86a6a 100755 --- a/vofs/setup.py +++ b/vofs/setup.py @@ -100,7 +100,6 @@ def run_tests(self): 'Programming Language :: Python', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6' diff --git a/vofs/tox.ini b/vofs/tox.ini index 9d0593dc0..a781d996c 100644 --- a/vofs/tox.ini +++ b/vofs/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py33, py34, py35, py36 +envlist = py27, py34, py35, py36 skip_missing_interpreters = True [testenv] diff --git a/vos/setup.cfg b/vos/setup.cfg index 7131f0a57..d5fea747e 100644 --- a/vos/setup.cfg +++ b/vos/setup.cfg @@ -51,7 +51,7 @@ edit_on_github = False github_project = opencadc/vostools install_requires = html2text>=2016.5.29 cadcutils>=1.1.2 future # version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440) -version = 3.0.2 +version = 3.0.3 [entry_points] vcat = vos.commands.vcat:vcat diff --git a/vos/setup.py b/vos/setup.py index f3a271b97..86cc86a6a 100755 --- a/vos/setup.py +++ b/vos/setup.py @@ -100,7 +100,6 @@ def run_tests(self): 'Programming Language :: Python', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6' diff --git a/vos/tox.ini b/vos/tox.ini index 9d0593dc0..a781d996c 100644 --- a/vos/tox.ini +++ b/vos/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py33, py34, py35, py36 +envlist = py27, py34, py35, py36 skip_missing_interpreters = True [testenv] diff --git a/vos/vos/tests/test_commonparser.py b/vos/vos/tests/test_commonparser.py index 88eeb1d37..6167c1f8e 100644 --- a/vos/vos/tests/test_commonparser.py +++ b/vos/vos/tests/test_commonparser.py @@ -19,8 +19,8 @@ def test_all(self): sys.argv = ['myapp', '-w'] args = common_parser.parse_args() set_logging_level_from_args(args) - self.assertEquals(version, common_parser.version) - self.assertEquals(logging.WARNING, logging.getLogger('root').level) + self.assertEqual(version, common_parser.version) + self.assertEqual(logging.WARNING, logging.getLogger('root').level) # Remove all handlers associated with the root logger object. for handler in logging.root.handlers[:]: @@ -29,8 +29,8 @@ def test_all(self): sys.argv = ['myapp', '-d'] args = common_parser.parse_args() set_logging_level_from_args(args) - self.assertEquals(logging.DEBUG, logging.getLogger().level) - self.assertEquals(version, common_parser.version) + self.assertEqual(logging.DEBUG, logging.getLogger().level) + self.assertEqual(version, common_parser.version) common_parser = CommonParser() sys.argv = ['myapp', '--version'] diff --git a/vos/vos/tests/test_vos.py b/vos/vos/tests/test_vos.py index 2322422e0..fbe0351a1 100644 --- a/vos/vos/tests/test_vos.py +++ b/vos/vos/tests/test_vos.py @@ -264,23 +264,51 @@ def test_copy(self, computed_md5_mock): # time to test... vospaceLocation = 'vos://test/foo' osLocation = '/tmp/foo' + if os.path.isfile(osLocation): + os.remove(osLocation) # copy from vospace test_client.copy(vospaceLocation, osLocation) get_node_url_mock.assert_called_once_with(vospaceLocation, method='GET', cutout=None, view='data') computed_md5_mock.assert_called_once_with(osLocation) + assert not get_node_mock.called + + # repeat - local file and vospace file are now the same -> only + # get_node is called to get the md5 of remote file + get_node_url_mock.reset_mock() + computed_md5_mock.reset_mock() + get_node_mock.reset_mock() + test_client.copy(vospaceLocation, osLocation) + assert not get_node_url_mock.called + computed_md5_mock.assert_called_once_with(osLocation) + get_node_mock.assert_called_once_with(vospaceLocation) + + # change the content of local files to trigger a new copy + get_node_url_mock.reset_mock() + computed_md5_mock.reset_mock() + computed_md5_mock.side_effect = ['d002233', md5sum] + get_node_mock.reset_mock() + test_client.copy(vospaceLocation, osLocation) + get_node_url_mock.assert_called_once_with(vospaceLocation, + method='GET', + cutout=None, view='data') + computed_md5_mock.assert_called_with(osLocation) get_node_mock.assert_called_once_with(vospaceLocation) # copy to vospace when md5 sums are the same -> only update occurs get_node_url_mock.reset_mock() computed_md5_mock.reset_mock() + computed_md5_mock.side_effect = None + computed_md5_mock.return_value = md5sum test_client.copy(osLocation, vospaceLocation) mock_update.assert_called_once() assert not get_node_url_mock.called # make md5 different get_node_url_mock.reset_mock() + get_node_url_mock.return_value =\ + ['http://cadc.ca/test', 'http://cadc.ca/test'] computed_md5_mock.reset_mock() mock_update.reset_mock() props.get.side_effect = ['d00223344', md5sum] diff --git a/vos/vos/vos.py b/vos/vos/vos.py index 504813e24..d6c2c6771 100644 --- a/vos/vos/vos.py +++ b/vos/vos/vos.py @@ -1513,7 +1513,6 @@ def copy(self, source, destination, send_md5=False, disposition=False): source_md5 = None get_node_url_retried = False content_disposition = None - if source[0:4] == "vos:": if destination is None: # Set the destination, initially, to the same directory as @@ -1542,7 +1541,22 @@ def copy(self, source, destination, send_md5=False, disposition=False): view = 'data' cutout = None check_md5 = True - source_md5 = self.get_node(source).props.get('MD5', ZERO_MD5) + if os.path.isfile(destination): + # check if the local file is up to date before doing the + # transfer + source_md5 = \ + self.get_node(source).props.get('MD5', ZERO_MD5) + destination_md5 =\ + md5_cache.MD5Cache.compute_md5(destination) + if source_md5 == destination_md5: + logger.info( + 'copy: src and dest are already the same') + destination_size = os.stat(destination).st_size + if disposition: + return None + if send_md5: + return destination_md5 + return destination_size get_urls = self.get_node_url(source, method='GET', cutout=cutout, view=view)