Refactored vos to facilitate integration with new storage (#151)

* Re-factored to facilitate new storage implementation
* Moved move to async transfer end point. Cleaned up code
* Made error messages more user friendly

vos/setup.cfg

@@ -48,9 +48,9 @@ license = AGPLv3
 url = https://www.canfar.net/en/docs/storage
 edit_on_github = False
 github_project = opencadc/vostools
-install_requires = html2text>=2016.5.29 cadcutils>=1.1.20 future aenum
+install_requires = html2text>=2016.5.29 cadcutils>=1.1.26 future aenum
 # version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440)
-version = 3.2b
+version = 3.2c
 
 
 [entry_points]

vos/vos/tests/test_vos.py

@@ -357,6 +357,8 @@ def test_copy(self, computed_md5_mock):
         get_node_url_mock.reset_mock()
         computed_md5_mock.reset_mock()
         get_node_mock.reset_mock()
+        props.reset_mock()
+        props.get.return_value = md5sum
         test_client.copy(vospaceLocation, osLocation)
         assert not get_node_url_mock.called
         computed_md5_mock.assert_called_once_with(osLocation)
@@ -365,6 +367,7 @@ def test_copy(self, computed_md5_mock):
         # change the content of local files to trigger a new copy
         get_node_url_mock.reset_mock()
         get_node_mock.reset_mock()
+
         computed_md5_mock.reset_mock()
         computed_md5_mock.return_value = 'd002233'
         response.iter_content.return_value = BytesIO(file_content)
@@ -375,6 +378,22 @@ def test_copy(self, computed_md5_mock):
         computed_md5_mock.assert_called_with(osLocation)
         get_node_mock.assert_called_once_with(vospaceLocation, force=True)
 
+        # change the content of local files to trigger a new copy
+        get_node_url_mock.reset_mock()
+        get_node_url_mock.return_value = \
+            ['https://mysite.com/node/node123/cutout']
+        computed_md5_mock.reset_mock()
+        computed_md5_mock.return_value = 'd002233'
+        # computed_md5_mock.side_effect = ['d002233', md5sum]
+        get_node_mock.reset_mock()
+        response.iter_content.return_value = BytesIO(file_content)
+        session.get.return_value = response
+        test_client.get_session = Mock(return_value=session)
+        test_client.copy('{}{}'.format(vospaceLocation,
+                                       '[1][10:60]'), osLocation)
+        get_node_url_mock.assert_called_once_with(
+            vospaceLocation, method='GET', cutout='[1][10:60]', view='cutout')
+
         # copy to vospace when md5 sums are the same -> only update occurs
         get_node_url_mock.reset_mock()
         computed_md5_mock.reset_mock()
@@ -486,38 +505,35 @@ def test_copy(self, computed_md5_mock):
 
     # patch sleep to stop the test from sleeping and slowing down execution
     @patch('vos.vos.time.sleep', MagicMock(), create=True)
-    @patch('vos.vos.VOFile')
-    def test_transfer_error(self, mock_vofile):
-        vofile = MagicMock()
-        mock_vofile.return_value = vofile
+    def test_transfer_error(self):
+        session = Mock()
+        conn_mock = MagicMock(spec=Connection)
+        conn_mock.session.return_value = session
+        end_point_mock = Mock(session=session)
 
         vospace_url = 'https://somevospace.server/vospace'
-        session = Mock()
+
         session.get.side_effect = [Mock(text='COMPLETED'),
                                    Mock(text='COMPLETED')]
-
-        test_client = Client()
-
-        # use the mocked connection instead of the real one
-        test_client.get_session = Mock(return_value=session)
+        test_transfer = vos.Transfer(end_point_mock)
 
         # job successfully completed
-        self.assertFalse(test_client.get_transfer_error(
+        self.assertFalse(test_transfer.get_transfer_error(
             vospace_url + '/results/transferDetails', 'vos://vospace'))
         session.get.assert_called_with(vospace_url + '/phase',
-                                       allow_redirects=False)
+                                       allow_redirects=True)
 
         # job suspended
         session.reset_mock()
         session.get.side_effect = [Mock(text='COMPLETED'),
                                    Mock(text='ABORTED')]
         with self.assertRaises(OSError):
-            test_client.get_transfer_error(
+            test_transfer.get_transfer_error(
                 vospace_url + '/results/transferDetails', 'vos://vospace')
         # check arguments for session.get calls
         self.assertEqual(
-            [call(vospace_url + '/phase', allow_redirects=False),
-             call(vospace_url + '/phase', allow_redirects=False)],
+            [call(vospace_url + '/phase', allow_redirects=True),
+             call(vospace_url + '/phase', allow_redirects=True)],
             session.get.call_args_list)
 
         # job encountered an internal error
@@ -526,10 +542,10 @@ def test_transfer_error(self, mock_vofile):
                                    Mock(text='ERROR'),
                                    Mock(text='InternalFault')]
         with self.assertRaises(OSError):
-            test_client.get_transfer_error(
+            test_transfer.get_transfer_error(
                 vospace_url + '/results/transferDetails', 'vos://vospace')
-        self.assertEqual([call(vospace_url + '/phase', allow_redirects=False),
-                          call(vospace_url + '/phase', allow_redirects=False),
+        self.assertEqual([call(vospace_url + '/phase', allow_redirects=True),
+                          call(vospace_url + '/phase', allow_redirects=True),
                           call(vospace_url + '/error')],
                          session.get.call_args_list)
 
@@ -541,13 +557,60 @@ def test_transfer_error(self, mock_vofile):
                                    Mock(
                                        text="Unsupported link target: " +
                                             link_file)]
-        self.assertEqual(link_file, test_client.get_transfer_error(
+        self.assertEqual(link_file, test_transfer.get_transfer_error(
             vospace_url + '/results/transferDetails', 'vos://vospace'))
-        self.assertEqual([call(vospace_url + '/phase', allow_redirects=False),
-                          call(vospace_url + '/phase', allow_redirects=False),
+        self.assertEqual([call(vospace_url + '/phase', allow_redirects=True),
+                          call(vospace_url + '/phase', allow_redirects=True),
                           call(vospace_url + '/error')],
                          session.get.call_args_list)
 
+    def test_transfer(self):
+        session = Mock()
+        redirect_response = Mock()
+        redirect_response.status_code = 303
+        redirect_response.headers = \
+            {'Location': 'https://transfer.host/transfer'}
+        response = Mock()
+        response.status_code = 200
+        response.text = (
+            '<?xml version="1.0" encoding="UTF-8"?>'
+            '<vos:transfer xmlns:vos="http://www.ivoa.net/xml/VOSpace/v2.0" '
+            'version="2.1">'
+            '<vos:target>vos://some.host~vault/abc</vos:target>'
+            '<vos:direction>pullFromVoSpace</vos:direction>'
+            '<vos:protocol uri="ivo://ivoa.net/vospace/core#httpsget">'
+            '<vos:endpoint>https://transfer.host/transfer/abc</vos:endpoint>'
+            '<vos:securityMethod '
+            'uri="ivo://ivoa.net/sso#tls-with-certificate" />'
+            '</vos:protocol>'
+            '<vos:keepBytes>true</vos:keepBytes>'
+            '</vos:transfer>')
+        session.post.return_value = redirect_response
+        session.get.return_value = response
+        conn_mock = MagicMock(spec=Connection)
+        conn_mock.session.return_value = session
+        end_point_mock = Mock(session=session)
+        test_transfer = vos.Transfer(end_point_mock)
+        protocols = test_transfer.transfer(
+            'https://some.host/service', 'vos://abc', 'pullFromVoSpace')
+        assert protocols == ['https://transfer.host/transfer/abc']
+
+        session.reset_mock()
+        session.post.return_value = Mock(status_code=404)
+        with self.assertRaises(OSError) as e:
+            test_transfer.transfer(
+                'https://some.host/service', 'vos://abc',
+                'pullFromVoSpace')
+            assert 'File not found: vos://abc' == str(e)
+
+        session.reset_mock()
+        session.post.return_value = Mock(status_code=500)
+        with self.assertRaises(OSError) as e:
+            test_transfer.transfer(
+                'https://some.host/service', 'vos://abc',
+                'pullFromVoSpace')
+            assert 'Failed to get transfer service response.' == str(e)
+
     def test_add_props(self):
         old_node = Node(ElementTree.fromstring(NODE_XML))
         old_node.uri = 'vos:sometest'