Copy not using md5 before transfer (#100)

* Fixed md5 check before copy file from vospace

.travis.yml

@@ -2,7 +2,6 @@ language: python
 
 python:
     - 2.7
-    - 3.3
     - 3.4
     - 3.5
     - 3.6

vofs/setup.cfg

@@ -49,9 +49,9 @@ license = AGPLv3
 url = http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/vospace
 edit_on_github = False
 github_project = opencadc/vostools
-install_requires = vos>=3.0 BitVector>=3.4.4,<4.0 fusepy>=2.0.4
+install_requires = vos>=3.0.3 BitVector>=3.4.4,<4.0 fusepy>=2.0.4
 # version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440)
-version = 3.0.1
+version = 3.0.2
 
 [entry_points]
 #astropy-package-template-example = packagename.example_mod:main

vofs/setup.py

@@ -100,7 +100,6 @@ def run_tests(self):
         'Programming Language :: Python',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6'

vofs/tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27, py33, py34, py35, py36
+envlist = py27, py34, py35, py36
 skip_missing_interpreters = True
 
 [testenv]

vos/setup.cfg

@@ -51,7 +51,7 @@ edit_on_github = False
 github_project = opencadc/vostools
 install_requires = html2text>=2016.5.29 cadcutils>=1.1.2 future
 # version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440)
-version = 3.0.2
+version = 3.0.3
 
 [entry_points]
 vcat = vos.commands.vcat:vcat

vos/setup.py

@@ -100,7 +100,6 @@ def run_tests(self):
         'Programming Language :: Python',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6'

vos/tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27, py33, py34, py35, py36
+envlist = py27, py34, py35, py36
 skip_missing_interpreters = True
 
 [testenv]

vos/vos/tests/test_commonparser.py

@@ -19,8 +19,8 @@ def test_all(self):
         sys.argv = ['myapp', '-w']
         args = common_parser.parse_args()
         set_logging_level_from_args(args)
-        self.assertEquals(version, common_parser.version)
-        self.assertEquals(logging.WARNING, logging.getLogger('root').level)
+        self.assertEqual(version, common_parser.version)
+        self.assertEqual(logging.WARNING, logging.getLogger('root').level)
 
         # Remove all handlers associated with the root logger object.
         for handler in logging.root.handlers[:]:
@@ -29,8 +29,8 @@ def test_all(self):
         sys.argv = ['myapp', '-d']
         args = common_parser.parse_args()
         set_logging_level_from_args(args)
-        self.assertEquals(logging.DEBUG, logging.getLogger().level)
-        self.assertEquals(version, common_parser.version)
+        self.assertEqual(logging.DEBUG, logging.getLogger().level)
+        self.assertEqual(version, common_parser.version)
 
         common_parser = CommonParser()
         sys.argv = ['myapp', '--version']

vos/vos/tests/test_vos.py

@@ -264,23 +264,51 @@ def test_copy(self, computed_md5_mock):
         # time to test...
         vospaceLocation = 'vos://test/foo'
         osLocation = '/tmp/foo'
+        if os.path.isfile(osLocation):
+            os.remove(osLocation)
         # copy from vospace
         test_client.copy(vospaceLocation, osLocation)
         get_node_url_mock.assert_called_once_with(vospaceLocation,
                                                   method='GET',
                                                   cutout=None, view='data')
         computed_md5_mock.assert_called_once_with(osLocation)
+        assert not get_node_mock.called
+
+        # repeat - local file and vospace file are now the same -> only
+        # get_node is called to get the md5 of remote file
+        get_node_url_mock.reset_mock()
+        computed_md5_mock.reset_mock()
+        get_node_mock.reset_mock()
+        test_client.copy(vospaceLocation, osLocation)
+        assert not get_node_url_mock.called
+        computed_md5_mock.assert_called_once_with(osLocation)
+        get_node_mock.assert_called_once_with(vospaceLocation)
+
+        # change the content of local files to trigger a new copy
+        get_node_url_mock.reset_mock()
+        computed_md5_mock.reset_mock()
+        computed_md5_mock.side_effect = ['d002233', md5sum]
+        get_node_mock.reset_mock()
+        test_client.copy(vospaceLocation, osLocation)
+        get_node_url_mock.assert_called_once_with(vospaceLocation,
+                                                  method='GET',
+                                                  cutout=None, view='data')
+        computed_md5_mock.assert_called_with(osLocation)
         get_node_mock.assert_called_once_with(vospaceLocation)
 
         # copy to vospace when md5 sums are the same -> only update occurs
         get_node_url_mock.reset_mock()
         computed_md5_mock.reset_mock()
+        computed_md5_mock.side_effect = None
+        computed_md5_mock.return_value = md5sum
         test_client.copy(osLocation, vospaceLocation)
         mock_update.assert_called_once()
         assert not get_node_url_mock.called
 
         # make md5 different
         get_node_url_mock.reset_mock()
+        get_node_url_mock.return_value =\
+            ['http://cadc.ca/test', 'http://cadc.ca/test']
         computed_md5_mock.reset_mock()
         mock_update.reset_mock()
         props.get.side_effect = ['d00223344', md5sum]

vos/vos/vos.py

@@ -1513,7 +1513,6 @@ def copy(self, source, destination, send_md5=False, disposition=False):
         source_md5 = None
         get_node_url_retried = False
         content_disposition = None
-
         if source[0:4] == "vos:":
             if destination is None:
                 # Set the destination, initially, to the same directory as
@@ -1542,7 +1541,22 @@ def copy(self, source, destination, send_md5=False, disposition=False):
                 view = 'data'
                 cutout = None
                 check_md5 = True
-                source_md5 = self.get_node(source).props.get('MD5', ZERO_MD5)
+                if os.path.isfile(destination):
+                    # check if the local file is up to date before doing the
+                    # transfer
+                    source_md5 = \
+                        self.get_node(source).props.get('MD5', ZERO_MD5)
+                    destination_md5 =\
+                        md5_cache.MD5Cache.compute_md5(destination)
+                    if source_md5 == destination_md5:
+                        logger.info(
+                            'copy: src and dest are already the same')
+                        destination_size = os.stat(destination).st_size
+                        if disposition:
+                            return None
+                        if send_md5:
+                            return destination_md5
+                        return destination_size
 
             get_urls = self.get_node_url(source, method='GET', cutout=cutout,
                                          view=view)