Merge pull request #87 from andamian/hotfix

Hotfix

.travis.yml

@@ -6,6 +6,7 @@ python:
     - 3.3
     - 3.4
     - 3.5
+    - 3.6
 
 # Setting sudo to false opts in to Travis-CI container-based builds.
 sudo: false
@@ -18,7 +19,7 @@ sudo: false
 addons:
     apt:
         packages:
-            - libfuse-dev
+             - libfuse-dev
 #            - graphviz
 #            - texlive-latex-extra
 #            - dvipng
@@ -54,15 +55,15 @@ matrix:
         #  env: SETUP_CMD='test'
 
 install:
-   - cd vos; pip install -r dev_requirements.txt; cd ..;
-   - cd vofs; pip install -r dev_requirements.txt; cd ..;
+   - pip install -U pip
+   - for i in $(ls -d */); do cd $i; pip install -r dev_requirements.txt; cd ..; done
+   - pip install coveralls
 
 script:
-   - cd vos; python setup.py $SETUP_CMD || exit -1; cd ..
-   - cd vofs; python setup.py $SETUP_CMD || exit -1
+    - for i in $(ls -d */); do cd $i; pytest -v --cov $i || exit 1; cd ..; done
 
 after_success:
     # If coveralls.io is set up for this package, uncomment the line
     # below and replace "packagename" with the name of your package.
     # The coveragerc file may be customized as needed for your package.
-    # - if [[ $SETUP_CMD == 'coverage' ]]; then coveralls; fi
+    - if [[ $SETUP_CMD == 'coverage' ]]; then coverage combine $(ls -d */.coverage) || exit 1; coveralls; fi

README.rst

@@ -0,0 +1,15 @@
+VOS - VOSpace tools
+
+
+.. image:: https://img.shields.io/pypi/pyversions/vps.svg
+    :target: https://pypi.python.org/pypi/vos
+
+.. image:: https://img.shields.io/travis/opencadc/vos2tools/master.svg   
+    :target: https://travis-ci.org/opencadc/vostools?branch=master 
+
+.. image:: https://img.shields.io/coveralls/opencadc/vostools/master.svg   
+    :target: https://coveralls.io/github/opencadc/vostools?branch=master 
+
+.. image:: https://img.shields.io/github/contributors/opencadc/vostools.svg
+    :target: https://github.com/opencadc/vostools/graphs/contributors
+

vofs/README.rst

@@ -1,6 +1,9 @@
 DOCUMENTATION
 =============
 
+.. image:: https://img.shields.io/pypi/v/vofs.svg   
+    :target: https://pypi.python.org/pypi/vofs
+
 vosfs is a python module that allows a VOSpace service to be used as a
 file system.
 

vofs/setup.py

@@ -4,25 +4,29 @@
 import glob
 import os
 import sys
+import imp
 from setuptools.command.test import test as TestCommand
 from setuptools import find_packages
 
 from setuptools import setup
 
+import distutils.cmd
+import distutils.log
+import subprocess
 
 # read the README.rst file and return as string.
 def readme():
     with open('README.rst') as r_obj:
         return r_obj.read()
 
-
 # Get some values from the setup.cfg
 try:
     from ConfigParser import ConfigParser
 except ImportError:
     from configparser import ConfigParser
 
 conf = ConfigParser()
+conf.optionxform=str
 conf.read(['setup.cfg'])
 metadata = dict(conf.items('metadata'))
 
@@ -33,15 +37,12 @@ def readme():
 LICENSE = metadata.get('license', 'unknown')
 URL = metadata.get('url', 'http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca')
 
-# Get the long description from the package's README.rst
-LONG_DESCRIPTION = readme()
-
 # VERSION should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386)
 VERSION = metadata.get('version', 'none')
 
 # generate the version file
 with open(os.path.join(PACKAGENAME, 'version.py'), 'w') as f:
-    f.write('version = \'{}\''.format(VERSION))
+    f.write('version = \'{}\''.format(VERSION))	
 
 # Treat everything in scripts except README.rst as a script to be installed
 scripts = [fname for fname in glob.glob(os.path.join('scripts', '*'))
@@ -55,7 +56,6 @@ def readme():
     entry_points['console_scripts'].append('{0} = {1}'.format(entry_point[0],
                                                               entry_point[1]))
 
-
 # add the --cov option to the test command
 class PyTest(TestCommand):
     """class py.test for the testing
@@ -73,7 +73,6 @@ def run_tests(self):
         err_no = pytest.main(self.pytest_args)
         sys.exit(err_no)
 
-
 # Note that requires and provides should not be included in the call to
 # ``setup``, since these are now deprecated. See this link for more details:
 # https://groups.google.com/forum/#!topic/astropy-dev/urYO8ckB2uM
@@ -87,12 +86,26 @@ def run_tests(self):
       author_email=AUTHOR_EMAIL,
       license=LICENSE,
       url=URL,
-      long_description=LONG_DESCRIPTION,
+      long_description=readme(),
       zip_safe=False,
       use_2to3=False,
       setup_requires=['pytest-runner'],
       entry_points=entry_points,
+      python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, <4',
       packages=find_packages(),
       package_data={PACKAGENAME: ['data/*', 'tests/data/*', '*/data/*', '*/tests/data/*']},
-      cmdclass={'coverage': PyTest}
-      )
+      classifiers=[
+        'Natural Language :: English',
+        'License :: OSI Approved :: GNU Affero General Public License v3',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6'
+      ],
+      cmdclass = {
+          'coverage': PyTest,
+      }
+)

vofs/test/scripts/vospace-mountvospace-atest.tcsh

@@ -84,6 +84,7 @@ set CONTAINER = $BASE/$TIMESTAMP
 set MOUNTPOINT=/tmp/vospace
 set MCONTAINER = "$MOUNTPOINT/$TIMESTAMP"
 
+echo "MCONTAINER: $MCONTAINER"
 echo "TIMESTAMP: $TIMESTAMP"
 
 echo -n "** checking base URI"
@@ -129,7 +130,6 @@ cat $ANONLOGFILE >> $LOGFILE  # since the original gets removed
 echo " [OK]"
 
 echo -n "mount vospace using certificate"
-echo "$MOUNTCMD $CERT --vospace="$BASE" --mountpoint=$MOUNTPOINT --cache_dir=$VOS_CACHE --log=$LOGFILE -d"
 $MOUNTCMD $CERT --vospace="$BASE" --mountpoint=$MOUNTPOINT --cache_dir=$VOS_CACHE --log=$LOGFILE -d >& /dev/null || echo " [FAIL]" && exit -1
 sleep 3
 ls $MOUNTPOINT >& /dev/null || echo [FAIL] && exit -1
@@ -181,14 +181,40 @@ echo -n "delete non-existent data node "
 rm $MCONTAINER/something.png >& /dev/null && echo " [FAIL]" && exit -1
 echo " [OK]"
 
+echo -n "create-delete-recreate test"
+# create a file, then update and at the same time delete it. The files should be gone
+touch $MCONTAINER/recreate.png || echo " [FAIL]" && exit -1
+cp -rf $thisDir/something.png $MCONTAINER/recreate.png >& /dev/null & rm $MCONTAINER/recreate.png || echo " [FAIL]" && exit -1
+sleep 3 
+if (! -f $MCONTAINER/recreate.png) then
+    echo " [FAIL]" && exit -1
+endif
+echo " [OK]"
+
+echo -n "rename container and then create a new container with the old name"
+mkdir $MCONTAINER/olddir || echo " [FAIL]" && exit -1
+echo -n " ."
+cp $thisDir/something.png $MCONTAINER/olddir/something.png || echo " [FAIL]" && exit -1
+echo -n "."
+mv $MCONTAINER/olddir $MCONTAINER/newdir || echo " [FAIL]" && exit -1
+echo -n "."
+ls $CONTAINER/newdir/something.png >& /dev/null && echo " [FAIL]" && exit -1
+echo -n "."
+mkdir $MCONTAINER/olddir || echo " [FAIL]" && exit -1
+echo -n "."
+cp $thisDir/something.png $MCONTAINER/olddir/something.png || echo " [FAIL]" && exit -1
+echo -n "."
+diff $MCONTAINER/newdir/something.png $MCONTAINER/olddir/something.png || echo " [FAIL]" && exit -1
+echo " [OK]"
+
 # --- test exceeding the local cache ---
-echo -n "copy cache test data to container"
+echo -n "copy cache test data to container "
 rm foo.dat >& /dev/null
 cat /dev/zero | head -c $CACHETEST_FSIZE_BYTES /dev/zero > foo.dat
 ls -l foo.dat
 foreach i ( `seq $CACHETEST_NFILES` )
     echo -n "."
-    echo "$CPCMD $CERT foo.dat $CONTAINER/foo$i.dat "
+    #echo "$CPCMD $CERT foo.dat $CONTAINER/foo$i.dat "
     $CPCMD $CERT foo.dat $CONTAINER/foo$i.dat >& /dev/null || echo " [FAIL]" && exit -1
 end
 rm foo.dat >& /dev/null

vofs/tox.ini

@@ -0,0 +1,7 @@
+[tox]
+envlist = py27, py33, py34, py35, py36
+skip_missing_interpreters = True
+
+[testenv]
+deps = -rdev_requirements.txt
+commands = python setup.py test

vofs/vofs/CadcCache.py

@@ -29,7 +29,7 @@
 _flush_thread_count = 0
 
 logger = logging.getLogger('cache')
-logger.setLevel(logging.ERROR)
+logger.setLevel(logging.INFO)
 if sys.version_info[1] > 6:
     logger.addHandler(logging.NullHandler())
 
@@ -210,7 +210,7 @@ def open(self, path, isNew, mustExist, ioObject, trustMetaData):
                     fileHandle.metaData = CacheMetaData(fileHandle.cacheMetaDataFile, None, None, None)
                     if fileHandle.metaData.getNumReadBlocks() == len(fileHandle.metaData.bitmap):
                         fileHandle.fullyCached = True
-                        fileHandle.fileSize = os.path.getsize(fileHandle.cacheMetaDataFile)
+                        fileHandle.fileSize = os.path.getsize(fileHandle.cacheDataFile)
                     else:
                         fileHandle.fullyCached = False
                         fileHandle.fileSize = fileHandle.metaData.size
@@ -418,14 +418,17 @@ def renameFile(self, oldPath, newPath):
             raise ValueError("Path '%s' is not an absolute path." % oldPath)
         if not os.path.isabs(newPath):
             raise ValueError("Path '%s' is not an absolute path." % newPath)
+        if os.path.isdir(newPath):
+            raise ValueError("Cannot rename '%s' file to '%s' directory." % (oldPath, newPath))
+        if oldPath == newPath:
+            return
         newDataPath = self.dataDir + newPath
         newMetaDataPath = self.metaDataDir + newPath
         oldDataPath = self.dataDir + oldPath
         oldMetaDataPath = self.metaDataDir + oldPath
         if os.path.isdir(oldDataPath):
-            raise ValueError("Path '%s' is a directory." % oldDataPath)
-        if os.path.isdir(oldMetaDataPath):
-            raise ValueError("Path '%s' is a directory." % oldMetaDataPath)
+            self.renameDir(oldPath, newPath)
+            return
 
         with self.cacheLock:
             # Make sure the new directory exists.
@@ -440,14 +443,16 @@ def renameFile(self, oldPath, newPath):
 
             try:
                 existingFileHandle = self.fileHandleDict[oldPath]
-                # If the file is active, rename its files with the lock held.
+                # If the file is active, rename its files with the lock held
+                # and remove the old reference from the file handle dictionary
+                # to allow subsequent uses of the old path
                 with existingFileHandle.fileLock:
                     Cache.atomicRename((oldDataPath, newDataPath),
                                        (oldMetaDataPath, newMetaDataPath))
-                    existingFileHandle.cacheDataFile = \
-                        os.path.abspath(newDataPath)
-                    existingFileHandle.cacheMetaDataFile = \
-                        os.path.abspath(newMetaDataPath)
+                    existingFileHandle.path = newPath
+                    existingFileHandle.cacheDataFile = newDataPath
+                    existingFileHandle.cacheMetaDataFile = newMetaDataPath
+                    del self.fileHandleDict[oldPath]
             except KeyError:
                 # The file is not active, rename the files but there is no
                 # data structure to lock or fix.
@@ -514,6 +519,7 @@ def renameDir(self, oldPath, newPath):
             finally:
                 for fh in lockedList:
                     if renamed:
+                        del self.fileHandleDict[fh.path]
                         # Change the data file name and meta data file name in
                         # the file handle.
                         start = len(oldDataPath)
@@ -523,6 +529,8 @@ def renameDir(self, oldPath, newPath):
                         fh.cacheMetaDataFile = os.path.abspath(
                             self.metaDataDir + newPath +
                             fh.cacheMetaDataFile[start:])
+                        start = len(oldPath)
+                        fh.path = os.path.abspath(newPath + fh.path[start:])
                     fh.fileLock.release()
 
     def getAttr(self, path):
@@ -533,18 +541,22 @@ def getAttr(self, path):
         open and has been modified.
         """
         # logger.debug("gettattr %s:" % path)
-
         with self.cacheLock:
             # Make sure the file state doesn't change in the middle.
             try:
                 fileHandle = self.fileHandleDict[path]
             except KeyError:
                 return None
             with fileHandle.fileLock:
+                try:
+                    f = os.stat(fileHandle.cacheDataFile)
+                except Exception as e:
+                    # error in accessing the cached version of the file. Remove from cache
+                    self.unlinkFile(fileHandle.cacheDataFile)
+                    return None
                 if fileHandle.fileModified:
                     # logger.debug("file modified: %s" %
                     #              fileHandle.fileModified)
-                    f = os.stat(fileHandle.cacheDataFile)
                     # logger.debug("size = %d:" % f.st_size)
                     return dict((name, getattr(f, name))
                                 for name in dir(f)
@@ -762,8 +774,6 @@ def __init__(self, path, cache, ioObject):
         with self.ioObject.cacheFileDescriptorLock:
             self.ioObject.cacheFileDescriptor = os.open(self.cacheDataFile,
                                                         os.O_RDWR | os.O_CREAT)
-            # Why is there an fstat here?
-            info = os.fstat(self.ioObject.cacheFileDescriptor)
 
         # logger.debug("Created a cache file descriptor.")
         # When cache locks and file locks need to be held at the same time,

vofs/vofs/tests/test_cadc_cache.py

@@ -591,6 +591,7 @@ def test_04_renameFile(self):
             # Rename an existing active file.
             with testCache.open("/dir1/dir2/file2", True, False, testIOProxy, False) as fh:
                 testCache.renameFile("/dir1/dir2/file2", "/dir1/dir3/file3")
+                self.assertEqual("/dir1/dir3/file3", fh.path)
                 self.assertEqual(fh.cacheDataFile, os.path.join(
                     testCache.dataDir, "dir1/dir3/file3"))
                 self.assertEqual(fh.cacheMetaDataFile, os.path.join(
@@ -634,9 +635,15 @@ def test_04_renameFile(self):
             with self.assertRaises(ValueError):
                 testCache.renameFile("/dir1/file", "dir2/file")
 
-            # Rename a directory. Should fail.
-            with self.assertRaises(ValueError):
+            # Rename a directory.
+            with testCache.open("/dir1/dir2/file", True, False, testIOProxy,
+                                False) as fh:
+                self.assertEquals(1, len(testCache.fileHandleDict))
                 testCache.renameFile("/dir1/dir2", "/dir3")
+                self.assertEqual(0, len(testCache.fileHandleDict))
+                self.assertEqual("/dir3/file", fh.path)
+                self.assertTrue("/dir3" in fh.cacheDataFile)
+                self.assertTrue("/dir3" in fh.cacheMetaDataFile)
 
             # Cause an error when the meta data file is rename. This should
             # raise an exception and not rename either file.
@@ -710,19 +717,6 @@ def test_04_renameFile2(self):
             with self.assertRaises(ValueError):
                 testCache.renameFile("/dir1/dir2", "/")
 
-            # Rename a file to a directory. For this one the meta data file is
-            # a directory.
-            with patch('os.path.isdir') as mockedisdir:
-                def returnFalse(arg):
-                    mockedisdir.side_effect = returnTrue
-                    return False
-
-                def returnTrue(arg):
-                    return True
-
-                mockedisdir.side_effect = returnFalse
-                with self.assertRaises(ValueError):
-                    testCache.renameFile("/something", "/something")
 
     @unittest.skipIf(skipTests, "Individual tests")
     def test_04_renameDir(self):

vofs/vofs/vofs.py

@@ -646,7 +646,10 @@ def rename(self, src, dest):
             result = self.client.move(src, dest)
             logger.debug(str(result))
             if result:
-                self.cache.renameFile(src, dest)
+                if os.path.isdir(src):
+                    self.cache.renamedir(src, dest)
+                else:
+                    self.cache.renameFile(src, dest)
                 return 0
             return -1
         except Exception as e: