Merge cd04d2c into 36357b3

vos/new-commands-README.md

@@ -0,0 +1,99 @@
+## vostools 3.1.0
+
+### Proposed command line (vcp, vrm) syntax 
+Initially the new commands will only be supported by the minoc service.
+#### vcp
+
+    usage: vcp [-h] [--certfile CERTFILE] [--token TOKEN] [--version] [-d]
+               [--vos-debug] [-v] [-w] [--exclude EXCLUDE] [--include INCLUDE]
+               [-i] [--overwrite] [--quick] [-L] [--ignore] [--head]
+               source [source ...] destination
+
+Copy files from source to destination.  A resource ID is required to determine the service to be used. The resource ID will reside in a configuration file. Example resource ID for vospace is ivo://cadc.nrc.ca/vault. Example resource ID for minoc ivo://cadc.nrc.ca/???/minoc, where ??? has yet to be defined. There will be multiple resource IDs for minoc, one per storage stie. 
+
+Positional Argument | Description        |
+----------------------------|----------------------|
+ Source                   | a URI or a local filename. URI will not support wild card, i.e. cadc:TEST/f\* will be invalid. Local filename will support wild card, e.g. *.fits is valid|
+ Destination           | a URI or a local filename. URI will not support wild card, i.e. cadc:TEST/f\* will be invalid. Local filename will support wild card, e.g. *.fits is valid|
+
+A URI is differentiated from a filename by '<scheme\>:'. When a URI is used to specify the destination, if the URI specifies a directory, it must end with a '/'. 
+
+Optional Arguments | Description     |
+----------------------------|------------------|
+-h, --help | show the help message and exit|
+--certfile CERTFILE | filename of your CADC X509 authentication certificate|
+--token TOKEN | authentication token string (alternative to certfile|
+--version | show program's version number and exit |
+-d, --debug| print on command debug messages.|
+--vos-debug| print on vos debug messages.|
+-v, --verbose | print verbose messges|
+-w, --warning | print warning messages only |
+--exclude EXCLUDE | skip files that match pattern (overrides include) |
+--include INCLUDE | only copy files that match pattern |
+-i, --interrogate | ask before overwriting files |
+--quick | assuming CANFAR VOSpace, only compatible with CANFAR VOSpace.|
+-L, --follow-links | follow symbolic links. Default is to not follow links. |
+--ignore | ignore errors and continue with recursive copy |
+--head | copy only the headers of a file from vospace. Might return an error if the server does not support the operation ona given file type. |
+
+Examples:
+To reduce clutter, optional arguments are not included in the examples.
+
+Put a local file foo.fits into cadc:TEST. 
+
+    vcp foot.fits cadc:TEST/
+
+Put all local files with '.fits' suffix into cadc:TEST
+
+    vcp *.fits cadc:TEST/
+
+Put a local file foo.fits into cadc:TEST as foobar.fits
+
+    vcp foo.fits cadc:TEST/foobar.fits
+
+Get a file foo.fits from cadc:TEST into the current directory
+
+    vcp cadc:TEST/foo.fits .
+
+Get a file foobar.fits from cadc:TEST into the current directory as foo.fits
+
+    vcp cadc:TEST/foobar.fits foo.fits
+
+Get a file foo.fits from cadc:TEST into the /tmp directory as foo.fits
+
+    vcp cadc:TEST/foo.fits /tmp
+
+#### vrm
+
+    usage: vrm [-h] [--certfile CERTFILE] [--token TOKEN] [--version] [-d]
+           [--vos-debug] [-v] [-w]
+           uri [uri ...]
+
+Remove one or more files identified by a URI. Wild card will not be supported, e.g. cadc:TEST/*.fits will be invalid.
+
+Positional Argument | Description        |
+----------------------------|----------------------|
+ uri    | a URI identifying the file to be removed |
+
+ Optional Arguments | Description     |
+----------------------------|------------------|
+-h, --help | show the help message and exit|
+--certfile CERTFILE | filename of your CADC X509 authentication certificate|
+--token TOKEN | authentication token string (alternative to certfile|
+--version | show program's version number and exit |
+-d, --debug| print on command debug messages.|
+--vos-debug| print on vos debug messages.|
+-v, --verbose | print verbose messges|
+-w, --warning | print warning messages only |
+
+Examples:
+To reduce clutter, optional arguments are not included in the examples.
+
+Remove file foo.fits from cadc:TEST. 
+
+    vrm cadc:TEST/foo.fits
+
+Remove files foo.fits, foobar.fits, goobar.fits from cadc:TEST
+
+    vrm cadc:TEST/foo.fits cadc:TEST/foobar.fits cadc:TEST/goobar.fits 
+

vos/setup.cfg

@@ -50,7 +50,7 @@ edit_on_github = False
 github_project = opencadc/vostools
 install_requires = html2text>=2016.5.29 cadcutils>=1.1.26 future aenum
 # version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440)
-version = 3.2c
+version = 3.2.newstorage
 
 
 [entry_points]

vos/setup.py

@@ -42,7 +42,7 @@ def readme():
 
 # generate the version file
 with open(os.path.join(PACKAGENAME, 'version.py'), 'w') as f:
-    f.write('version = \'{}\'\n'.format(VERSION))	
+    f.write('version = \'{}\'\n'.format(VERSION))
 
 # Treat everything in scripts except README.rst as a script to be installed
 scripts = [fname for fname in glob.glob(os.path.join('scripts', '*'))

vos/test/scripts/vospace-all.tcsh

@@ -30,8 +30,10 @@ echo "vospace-node-properties.tcsh"
 $THIS_DIR/vospace-node-properties.tcsh $args || echo "FAIL vospace-node-properties.tcsh" && exit -1
 echo "vospace-lock-atest.tcsh"
 $THIS_DIR/vospace-lock-atest.tcsh $args || echo "FAIL vospace-lock-atest.tcsh" && exit -1
-echo "vospace-cookie-atest.tcsh"
-$THIS_DIR/vospace-token-atest.tcsh $args || echo "FAIL vospace-cookie-atest.tcsh" && exit -1
+echo "vospace-token-atest.tcsh"
+$THIS_DIR/vospace-token-atest.tcsh $args || echo "FAIL vospace-token-atest.tcsh" && exit -1
+# echo "vospace-storage-inventory-client-atest.tcsh (In development)"
+# $THIS_DIR/vospace-storage-inventory-client-atest.tcsh || echo "FAIL vospace-storage-inventory-client-atest.tcsh"
 
 echo
 echo "*** all test sequences passed ***"

vos/test/scripts/vospace-storage-inventory-client-atest.tcsh

@@ -0,0 +1,141 @@
+#!/bin/tcsh -f
+
+date
+echo "###################"
+/bin/rm -rf ~/.config/cadc-registry
+
+if (! ${?VOSPACE_WEBSERVICE} ) then
+	echo "VOSPACE_WEBSERVICE env variable not set, use default WebService URL"
+    setenv VOSPACE_WEBSERVICE 'www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca'
+else
+	echo "WebService URL (VOSPACE_WEBSERVICE env variable): ${VOSPACE_WEBSERVICE}"
+endif
+
+if (! ${?CADC_TESTCERT_PATH} ) then
+	echo "CADC_TESTCERT_PATH env variable not set. Must point to the location of x509_CADCRegtest1.pem cert file"
+    exit -1
+else
+    set CERTFILE = "$CADC_TESTCERT_PATH/x509_CADCRegtest1.pem"
+	echo "cert file:  (CADC_TESTCERT_PATH env variable): $CERTFILE"
+endif
+
+if (! ${?CADC_DEBUG} ) then
+    set DEBUG_FLAG = ""
+else
+    echo "Debugging is on.  Expect much output."
+    set DEBUG_FLAG = "-d"
+endif
+
+if (! ${?TMPDIR} ) then
+        echo "TMPDIR env variable not set, using /tmp"
+        set TMPDIR = "/tmp"
+else
+        echo "Using ${TMPDIR} for temporary files"
+endif
+
+set SITE_1_RESOURCE_ID = "ivo://cadc.nrc.ca/minoc"
+set SET_SITE_1_RESOURCE_ID = " --resource-id ${SITE_1_RESOURCE_ID}"
+
+set THIS_DIR = `dirname $0`
+set THIS_DIR = `cd $THIS_DIR && pwd`
+
+# Username / password for getting tokens
+echo "Enter credentials for a VOSpace account in which we will perform tests."
+echo -n "CADC Username: "
+set username = $<
+echo -n "Password: "
+stty -echo
+set password = $<
+echo
+stty echo
+
+set DIFFCMD = "diff -q"
+
+set RMCMD = "vrm ${DEBUG_FLAG} ${SET_SITE_1_RESOURCE_ID}"
+set CPCMD = "vcp ${DEBUG_FLAG} ${SET_SITE_1_RESOURCE_ID}"
+set RMDIRCMD = "vrmdir ${DEBUG_FLAG}"
+
+set CERT = " --cert=$CERTFILE"
+set TOKEN = "--token ${TOKEN}"
+
+# using a test dir makes it easier to cleanup a bunch of old/failed tests
+set ROOT = "cadc:"
+set HOME = "${ROOT}CADCRegtest1"
+set BASE = "${HOME}/atest"
+
+set TIMESTAMP=`date +%Y-%m-%dT%H-%M-%S`
+set CONTAINER = "${BASE}/${TIMESTAMP}"
+
+echo
+echo "*** Starting test sequence ***"
+echo
+echo "** test container: ${CONTAINER}"
+echo
+echo "** test host: ${VOSPACE_WEBSERVICE}"
+echo
+echo "** test resource: ${SITE_1_RESOURCE_ID}"
+echo
+
+echo "-----------"
+echo -n "Copy file to existing container and non-existent data node "
+$CPCMD ${TOKEN} $THIS_DIR/something.png ${CONTAINER}/something.png || echo " [FAIL]" && exit -1
+echo " [OK]"
+
+# echo "-----------"
+# echo -n "Copy existing file from existing container "
+# $CPCMD $CERT $CONTAINER/something.png $THIS_DIR/something.png.2 || echo " [FAIL]" && exit -1
+# cmp $THIS_DIR/something.png $THIS_DIR/something.png.2 || echo " [FAIL]" && exit -1
+# echo " [OK]"
+
+# echo "-----------"
+# echo -n "Copy/overwrite existing data node "
+# $CPCMD $CERT $THIS_DIR/something.png.2 $CONTAINER/something.png || echo " [FAIL]" && exit -1
+# echo " [OK]"
+
+# echo -n "Upload check quick copy "
+# $CPCMD $CERT $THIS_DIR/something.png $CONTAINER/something.png.3|| echo " [FAIL]" && exit -1
+# echo " [OK]"
+
+# echo -n "Download check quick copy "
+# $CPCMD $CERT $CONTAINER/something.png.3 $THIS_DIR/something.png.3 || echo " [FAIL]" && exit -1
+# echo " [OK]"
+
+# echo -n "Compare check quick copy "
+# cmp $THIS_DIR/something.png $THIS_DIR/something.png.3 || echo " [FAIL]" && exit -1
+# \rm -f $THIS_DIR/something.png.3
+# echo " [OK]"
+
+# echo -n "Check pattern matched copy"
+# $CPCMD $CERT "$CONTAINER/something*" $TMPDIR || echo " [FAIL]" && exit -1
+# $CPCMD $CERT $THIS_DIR/something* $CONTAINER/pattern_dest || echo " [FAIL]" && exit -1
+# echo " [OK]"
+
+# Cutouts not supported yet.
+# jenkinsd 2020.02.11
+#
+# echo -n "Check pattern with cutout"
+# $CPCMD $CERT "$BASE/test*.fits[1:10,1:10]" $TMPDIR || echo " [FAIL]" && exit -1
+# echo " [OK]"
+
+# Not supported yet.
+# jenkinsd 2020.02.11
+#
+# echo -n "Copy empty files and should print message "
+# rm -f /tmp/zerosize.txt
+# touch /tmp/zerosize.txt
+# $CPCMD $CERT /tmp/zerosize.txt ${CONTAINER}/ || echo " [FAIL]" && exit -1
+# echo " [OK]"
+
+# echo -n "delete existing data node "
+# $RMCMD $CERT $CONTAINER/something.png  >& /dev/null || echo " [FAIL]" && exit -1
+# echo " [OK]"
+
+echo "-----------"
+echo -n "Cleanup "
+\rm -f $THIS_DIR/something.png.*
+echo " [OK]"
+
+echo
+echo "*** Test sequence passed ***"
+
+date

vos/test/scripts/vospace-token-atest.tcsh

@@ -17,13 +17,43 @@ if (! ${?VOSPACE_CONFIG_FILE} ) then
 else
     echo "Using VOSPACE_CONFIG_FILE: $VOSPACE_CONFIG_FILE"
 endif
+<<<<<<< HEAD
 
 if($#argv == 0) then
     set resources = "vault cavern"
     echo "Testing against default resources: $resources"
 else
     set resources = ($argv)
     echo "Testing against resources: $resources"
+=======
+echo " [OK]"
+
+echo -n "copy a file to scoped tree"
+$CPCMD --token="$TOKEN" $THIS_DIR/something.png $CONTAINER/B/ || echo " [FAIL]" && exit -1
+echo " [OK]"
+
+echo -n "check that the file got there"
+$LSCMD --token="$TOKEN" $CONTAINER/B | grep -q 'something.png' || echo " [FAIL]" && exit -1
+echo " [OK]"
+
+echo -n "create sub container with file in it"
+$MKDIRCMD --token="$TOKEN" $CONTAINER/B/test > /dev/null || echo " [FAIL]" && exit -1
+$CPCMD --token="$TOKEN" $THIS_DIR/something.png $CONTAINER/B/test/ || echo " [FAIL]" && exit -1
+$LSCMD --token="$TOKEN" $CONTAINER/B/test | grep -q 'something.png' || echo " [FAIL]" && exit -1
+echo " [OK]"
+
+echo -n "remove the file in the sub container"
+$RMCMD --token="$TOKEN" $CONTAINER/B/test/something.png || echo " [FAIL]" && exit -1
+echo " [OK]"
+
+echo -n "remove the sub container"
+$RMDIRCMD --token="$TOKEN" $CONTAINER/B/test || echo " [FAIL]" && exit -1
+
+echo -n "checking the remove was successful"
+set check_test = "`${LSCMD} --token='${TOKEN}' ${CONTAINER}/B | grep -q test`"
+if ( "${check_test}" != "" ) then
+    echo " [FAIL]" && exit -1
+>>>>>>> newstorage
 endif
 
 set ACCESS_PAGE=https://ws-cadc.canfar.net/ac/login

vos/vos/__init__.py

@@ -15,3 +15,4 @@
 
 """
 from .vos import Client, Connection, Node, VOFile  # noqa
+from .storage_inventory import Client as StorageInventoryClient  # noqa