S2804 (#154)

.travis.yml

@@ -5,21 +5,23 @@ python:
     - 3.4
     - 3.5
     - 3.6
+    - 3.7
+    - 3.8
 
 # Setting sudo to false opts in to Travis-CI container-based builds.
 sudo: false
 
 install:
     - pip install -U pip
-    - pip install twine
+    - if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then pip install twine; fi
     - for i in $(ls -d */); do cd $i; pip install -r dev_requirements.txt; cd ..; done
     - pip install coveralls
 
 script:
     - for i in $(ls -d */); do
          cd $i;
          pytest --cov $i || exit 1;
-         if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then
+         if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then
          flake8 -v $i || exit -1;
          fi;
          cd ..;
@@ -30,15 +32,15 @@ after_success:
     # below and replace "packagename" with the name of your package.
     # The coveragerc file may be customized as needed for your package.
     # Run coverage for all Python versions but submit only once - 3.5
-    - if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then coverage combine $(ls -d */.coverage) || exit 1; coveralls; fi
+    - if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then coverage combine $(ls -d */.coverage) || exit 1; coveralls; fi
 
 jobs:
   include:
       #- stage: intTest
-      #python: 3.5
+      #python: 3.7
       #script: ./.intTest
     - stage: publish
-      python: 3.4
+      python: 3.7
       deploy:
          provider: script
          script: ./.publish.sh

vofs/dev_requirements.txt

@@ -1,6 +1,6 @@
 -e ../vos
 -e . 
-pytest>=3.6
+pytest>=4.6
 pytest-cov>=2.5.1
 flake8>=3.4.1
 mock==2.0.0

vofs/setup.cfg

@@ -49,9 +49,9 @@ license = AGPLv3
 url = https://www.canfar.net/en/docs/storage 
 edit_on_github = False
 github_project = opencadc/vostools
-install_requires = vos>=3.0.3 BitVector>=3.4.4,<4.0 fusepy>=2.0.4
+install_requires = vos>=3.2a BitVector>=3.4.4,<4.0 fusepy>=2.0.4
 # version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440)
-version = 3.0.3
+version = 3.1a
 
 [entry_points]
 #astropy-package-template-example = packagename.example_mod:main

vofs/test/scripts/vospace-mountvospace-atest.tcsh

@@ -108,6 +108,7 @@ rm -fR $VOS_CACHE #clean the cache
 # For the anonymous mount test we use a separate temporary log
 echo -n "mount vospace anonymously"
 rm $ANONLOGFILE >& /dev/null
+echo "$MOUNTCMD --certfile=anonymous --mountpoint=$MOUNTPOINT --cache_dir=$VOS_CACHE --log=$ANONLOGFILE -d"
 $MOUNTCMD --certfile=anonymous --mountpoint=$MOUNTPOINT --cache_dir=$VOS_CACHE --log=$ANONLOGFILE -d >& /dev/null || echo " [FAIL]" && exit -1
 sleep 3
 ls $MOUNTPOINT >& /dev/null || echo " [FAIL]" && exit -1

vofs/tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27, py34, py35, py36
+envlist = py27, py34, py35, py36, py37, py38
 skip_missing_interpreters = True
 
 [testenv]

vofs/vofs/mountvofs.py

@@ -6,12 +6,14 @@
 import os
 import logging
 import getpass
+from six.moves.urllib.parse import urlparse
 
 from vos import vos
 from .version import version
 from .vofs import VOFS
 from .vofs import MyFuse
 from vos.commonparser import CommonParser, set_logging_level_from_args
+from vos import vosconfig
 
 DAEMON_TIMEOUT = 60
 
@@ -20,7 +22,12 @@ def mountvofs():
     parser = CommonParser(description='mount vospace as a filesystem.')
 
     # mountvofs specific options
-    parser.add_option("--vospace", help="the VOSpace to mount", default="vos:")
+    parser.add_option("--vospace",
+                      help="the VOSpace to mount. Default is vos (CADC vault "
+                           "service) but other VOSpaces can be used and "
+                           "referred to through their configured prefix (see "
+                           "vos-config command)",
+                      default="vos:/")
     parser.add_option("--mountpoint",
                       help="the mountpoint on the local filesystem",
                       default="/tmp/vospace")
@@ -95,7 +102,16 @@ def mountvofs():
     else:
         certfile = os.path.abspath(opt.certfile)
 
-    conn = vos.Connection(vospace_certfile=certfile, vospace_token=opt.token)
+    service_prefix = None  # default service prefix
+    if opt.vospace:
+        service_prefix = urlparse(opt.vospace).scheme
+    try:
+        resource_id = vosconfig.vos_config.get_resource_id(service_prefix)
+    except Exception as e:
+        raise RuntimeError(
+            'Cannot identify vospace service: {}'.format(str(e)))
+    conn = vos.Connection(resource_id=resource_id,
+                          vospace_certfile=certfile, vospace_token=opt.token)
     if opt.token:
         readonly = opt.readonly
         logger.debug("Got a token, connections should work")
@@ -115,7 +131,8 @@ def mountvofs():
         os.makedirs(mount)
     try:
         if platform == "darwin":
-            MyFuse(VOFS(root, opt.cache_dir, opt, conn=conn,
+            MyFuse(VOFS(root, opt.cache_dir, opt,
+                        vospace_certfile=certfile, vospace_token=opt.token,
                         cache_limit=opt.cache_limit,
                         cache_nodes=opt.cache_nodes,
                         cache_max_flush_threads=opt.max_flush_threads,
@@ -133,7 +150,8 @@ def mountvofs():
                    debug=opt.foreground,
                    foreground=opt.foreground)
         else:
-            MyFuse(VOFS(root, opt.cache_dir, opt, conn=conn,
+            MyFuse(VOFS(root, opt.cache_dir, opt,
+                        vospace_certfile=certfile, vospace_token=opt.token,
                         cache_limit=opt.cache_limit,
                         cache_nodes=opt.cache_nodes,
                         cache_max_flush_threads=opt.max_flush_threads,

vofs/vofs/tests/test_vofs.py

@@ -57,7 +57,7 @@ def readData(self, start, mandatory, optional):
 
 
 class TestVOFS(unittest.TestCase):
-    testMountPoint = "/tmp/testfs"
+    testMountPoint = "vos:tmp/testfs"
     testCacheDir = "/tmp/testCache"
 
     def __init__(self, *args, **kwargs):
@@ -298,17 +298,18 @@ def test_create(self):
         parentNode.groupread = True
         parentNode.groupwrite = True
         testfs.client.open = Mock()
-        testfs.getNode = Mock(side_effect=SideEffect({
-            ('/dir1/dir2/file',): node,
-            ('/dir1/dir2',): parentNode}, name="testfs.getNode"))
-        with patch('vos.vos.Client.create', Mock(return_value=node)):
-            with self.assertRaises(FuseOSError):
-                testfs.create(file, os.O_RDWR)
+        testfs.client.get_node = Mock(side_effect=SideEffect({
+            (self.testMountPoint+'/dir1/dir2/file',): node,
+            (self.testMountPoint+'/dir1/dir2',): parentNode},
+            name="testfs.getNode"))
+        testfs.client.create = Mock(return_value=node)
+        with self.assertRaises(OSError):
+            testfs.create(file, os.O_RDWR)
 
         testfs.getNode = Mock(side_effect=FuseOSError(errno=5))
-        with patch('vos.vos.Client.create', Mock(return_value=node)):
-            with self.assertRaises(FuseOSError):
-                testfs.create(file, os.O_RDWR)
+        testfs.client.create = Mock(return_value=node)
+        with self.assertRaises(FuseOSError):
+            testfs.create(file, os.O_RDWR)
 
         node.props.get = Mock(return_value=False)
         testfs = vofs.VOFS(self.testMountPoint, self.testCacheDir, opt)
@@ -317,8 +318,8 @@ def test_create(self):
         testfs.getNode = Mock(side_effect=SideEffect({
             ('/dir1/dir2/file',): node,
             ('/dir1/dir2',): parentNode}, name="testfs.getNode"))
-        with patch('vos.vos.Client.create', Mock(return_value=node)):
-            testfs.create(file, os.O_RDWR)
+        testfs.client.create = Mock(return_value=node)
+        testfs.create(file, os.O_RDWR)
         testfs.open.assert_called_once_with(file, os.O_WRONLY)
 
     @unittest.skipIf(skipTests, "Individual tests")
@@ -406,7 +407,7 @@ def test_unlink(self):
         testfs.unlink(path)
         testfs.getNode.assert_called_once_with(path, force=False, limit=1)
         testfs.cache.unlinkFile.assert_called_once_with(path)
-        testfs.client.delete.assert_called_once_with(path)
+        testfs.client.delete.assert_called_once_with(self.testMountPoint+path)
 
         for mock in mocks:
             mock.reset_mock()
@@ -430,7 +431,8 @@ def test_mkdir(self):
         testfs.client.mkdir = Mock()
         testfs.client.update = Mock()
         testfs.mkdir(path, stat.S_IRUSR)
-        testfs.client.mkdir.assert_called_once_with(path)
+        testfs.client.mkdir.assert_called_once_with('{}{}'.format(
+            self.testMountPoint, path))
 
         # Try to make a directory in a locked parent.
         testfs.client.mkdir.reset_mock()
@@ -456,7 +458,7 @@ def test_rmdir(self):
         node.name = "testNode"
         testfs.client.get_node = Mock(return_value=node)
         testfs.rmdir(path)
-        testfs.client.delete.assert_called_once_with(path)
+        testfs.client.delete.assert_called_once_with(self.testMountPoint+path)
 
         # Try deleting a node which is locked.
         node.props.get = Mock(side_effect=SideEffect({
@@ -676,7 +678,8 @@ def test_rename(self):
         testfs.get_node = Mock(side_effect=exceptions.NotFoundException())
         with self.assertRaises(Exception):
             self.assertEqual(testfs.rename(src, dest), -1)
-        testfs.client.move.assert_called_once_with(src, dest)
+        testfs.client.move.assert_called_once_with(self.testMountPoint+src,
+                                                   self.testMountPoint+dest)
         self.assertEqual(testfs.cache.renameFile.call_count, 0)
 
         # Rename throws an exception because the node is locked.
@@ -686,7 +689,8 @@ def test_rename(self):
             "the node is NodeLocked so won't work")
         with self.assertRaises(OSError):
             self.assertEqual(testfs.rename(src, dest), -1)
-        testfs.client.move.assert_called_once_with(src, dest)
+        testfs.client.move.assert_called_once_with(self.testMountPoint+src,
+                                                   self.testMountPoint+dest)
         self.assertEqual(testfs.cache.renameFile.call_count, 0)
 
     @unittest.skipIf(skipTests, "Individual tests")
@@ -842,7 +846,7 @@ def test_getNode(self):
         testfs.client.get_node = Mock(return_value=node)
         node = testfs.getNode(file, force=True, limit=10)
         testfs.client.get_node.assert_called_once_with(
-            file, force=True, limit=10)
+            self.testMountPoint+file, force=True, limit=10)
 
         err = OSError()
         err.errno = 1
@@ -1077,10 +1081,9 @@ def test_getSize(self):
     def test_MyFuse(self, mock_libfuse):
         # Tests the ctor to make sure the arguments are passed
         # correctly to the os
-        conn = MagicMock()
         mock_libfuse.fuse_main_real.return_value = False
         fuseops = fuse_operations()
-        MyFuse(VOFS(":vos", "/tmp/vos_", None, conn=conn,
+        MyFuse(VOFS("vos:", "/tmp/vos_", None,
                     cache_limit=100,
                     cache_max_flush_threads=3),
                "/tmp/vospace",
@@ -1094,7 +1097,7 @@ def test_MyFuse(self, mock_libfuse):
                                                        ctypes.sizeof(fuseops),
                                                        None)
 
-        MyFuse(VOFS(":vos", "/tmp/vos_", None, conn=conn,
+        MyFuse(VOFS("vos:", "/tmp/vos_", None,
                     cache_limit=100,
                     cache_max_flush_threads=3),
                "/tmp/vospace",
@@ -1112,9 +1115,8 @@ def test_MyFuse(self, mock_libfuse):
     @patch("vofs.vofs.FUSE.__init__")
     def test_readMyFuse(self, mock_fuse):
         mock_fuse.return_value = None
-        conn = MagicMock()
         buf = ctypes.create_string_buffer(4)
-        fuse = MyFuse(VOFS("vos:/anode", "/tmp/vos_", None, conn=conn,
+        fuse = MyFuse(VOFS("vos:/anode", "/tmp/vos_", None,
                            cache_limit=100,
                            cache_max_flush_threads=3),
                       "/tmp/vospace",
@@ -1136,9 +1138,8 @@ def test_readMyFuse(self, mock_fuse):
     @patch("vofs.vofs.FUSE.__init__")
     def test_writeMyFuse(self, mock_fuse):
         mock_fuse.return_value = None
-        conn = MagicMock()
         buf = ctypes.create_string_buffer(4)
-        fuse = MyFuse(VOFS("vos:/anode", "/tmp/vos_", None, conn=conn,
+        fuse = MyFuse(VOFS("vos:/anode", "/tmp/vos_", None,
                            cache_limit=100,
                            cache_max_flush_threads=3),
                       "/tmp/vospace",