merge with SVN tree

getCert

@@ -65,7 +65,6 @@ parser.add_option('--daysValid', type=int, default=10,
                    help='Number of days the cetificate should be valid (default: 10)')
 parser.add_option('--dest', default=os.path.join(os.getenv('HOME','/tmp'),".ssl/cadcproxy.pem"),
                     help="Location to store the proxy certifacte")
-
 (opt,args) = parser.parse_args()
 
 if __name__=='__main__':

mountvofs

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 """A FUSE based filesystem view of VOSpace."""
 
 from sys import argv, exit, platform
@@ -13,6 +13,7 @@ import os
 import vos
 from os import O_RDONLY, O_WRONLY, O_RDWR, O_APPEND
 import logging
+READBUF=8192
 
 def flag2mode(flags):
     md = {O_RDONLY: 'r', O_WRONLY: 'w', O_RDWR: 'w+'}
@@ -193,9 +194,8 @@ class VOFS(LoggingMixIn, Operations):
         node=self.getNode(path)
         w=self.client.open(node.uri,'w')
         os.lseek(fh,0,0)
-        bufsize=2**33
         while True:
-            buf=os.read(fh,bufsize)
+            buf=os.read(fh,READBUF)
             logging.debug("Writing %d bytes to %s" % (len(buf), path))
             if not buf:
                 break
@@ -257,7 +257,7 @@ class VOFS(LoggingMixIn, Operations):
         self.node[path]=node
         self.attr[path]=node.attr
         if self.node[path].isdir():
-            for node in self.node[path].nodeList:
+            for node in self.node[path].getNodeList():
                 npath=os.path.join(path,node.name)
                 self.node[npath]=node
                 self.attr[npath]=node.attr
@@ -270,7 +270,10 @@ class VOFS(LoggingMixIn, Operations):
     def getattr(self, path, fh=None):
         """Build some attributes for this file, we have to make-up some stuff"""
 	if not path in self.attr:
-            node=self.getNode(path)
+            try:
+                node=self.client.getNode(path,limit=0)
+            except IOError as e:
+                raise FuseOSError(e.errno)
 	    self.attr[path]=node.attr
             logging.debug("Got %s for %s" % (node, path))
         atime=self.attr.get('st_atime',time.time())
@@ -280,7 +283,10 @@ class VOFS(LoggingMixIn, Operations):
             ### the modification/change times are after the last access
             ### so we should access this VOSpace node again.
             logging.debug("Getting node details from getattr for path %s" % ( path))
-            node=self.getNode(path,force=True)
+	    try:
+               node=self.client.getNode(path,limit=0)
+	    except IOError as e:
+               raise FuseOSError(e.errno)
             self.attr[path]=node.attr
 	    #self.attr[path].update(self.getNode(path).attr)
         return self.attr[path]
@@ -370,7 +376,7 @@ class VOFS(LoggingMixIn, Operations):
             r=self.client.open(path,mode=os.O_RDONLY,view="data")
             fpos=0
             while True:
-                buf=r.read(2**(10+10))
+                buf=r.read(READBUF)
                 logging.debug("Read buffer length %s from %s" % ( len(buf), self.getNode(self.getPath(fh)).name)) 
                 if not buf:
                     break
@@ -394,7 +400,7 @@ class VOFS(LoggingMixIn, Operations):
 
     def readdir(self, path, fh):
         """Send a list of entried in this directory"""
-        return ['.','..'] + [e.name.encode('utf-8') for e in self.getNode(path,force=True).nodeList ]
+        return ['.','..'] + [e.name.encode('utf-8') for e in self.getNode(path,force=True).getNodeList() ]
 
     def release(self, path, fh):
         """Close the file, but if this was a holding spot for writes, then write the file to the node"""
@@ -407,7 +413,7 @@ class VOFS(LoggingMixIn, Operations):
                 w=self.client.open(path,os.O_WRONLY)
                 os.lseek(fh,0,0)
                 while True:
-                    buf=os.read(fh,2**(10+10))
+                    buf=os.read(fh,READBUF)
                     logging.debug("Writing %d bytes to %s" % (len(buf), path))
                     if not buf:
                         break
@@ -515,7 +521,7 @@ class VOFS(LoggingMixIn, Operations):
         node=self.getNode(path,force=True)
         #if not node.isdir():
         #    raise FuseOSError(ENOTDIR)
-        #if len(node.nodeList)>0:
+        #if len(node.getNodeList())>0:
         #    raise FuseOSError(ENOTEMPTY)
         fname=os.path.normpath(self.cache_dir+path)
         if os.access(fname,os.F_OK):
@@ -540,7 +546,7 @@ class VOFS(LoggingMixIn, Operations):
         sfs['f_blocks']=int(bytes/block_size)
         sfs['f_bfree']=int(free/block_size)
         sfs['f_bavail']=int(free/block_size)
-        sfs['f_files']=len(node.nodeList)
+        sfs['f_files']=len(node.getNodeList())
         sfs['f_ffree']=2*10
         sfs['f_favail']=2*10
         sfs['f_flags']=0
@@ -557,9 +563,8 @@ class VOFS(LoggingMixIn, Operations):
             except:
                 raise FuseOSError(EIO)
             fpos=0
-            bufsize=4096
             while True:
-                buf=r.read(bufsize)
+                buf=r.read(READBUF)
                 if not buf:
                     break
                 chunk=min(length-fpos,bufsize) 
@@ -630,6 +635,7 @@ if __name__ == "__main__":
     parser.add_option("--log",action="store",help="File to store debug log to",default="/tmp/vos.err")
     parser.add_option("--cache_limit",action="store",type=int,help="upper limit on local diskspace to use for file caching",default=50*2**(10+10+10))
     parser.add_option("--cache_dir",action="store",help="local directory to use for file caching",default=os.getenv('HOME',default='.'))
+    parser.add_option("--certfile",help="location of your CADC security certificate file",default=os.path.join(os.getenv("HOME","."),".ssl/cadcproxy.pem"))
 
     (opt,args)=parser.parse_args()
     formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
@@ -643,7 +649,7 @@ if __name__ == "__main__":
         logging.basicConfig(level=logging.ERROR,format="vofs:%(module)s.%(funcName)s %(message)s",filename=opt.log)
 
     logging.debug("Checking connetion to VOSpace ")
-    conn=vos.Connection()
+    conn=vos.Connection(certfile=opt.certfile)
     logging.debug("Got a certificate, connections should work")
 
     root = opt.vospace

vcp

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 """copy files from / to vospace directly without using the FUSE layer"""
 
 def isdir(filename):

vls

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 
 """vls:  list the contents of a voSpace"""
 import vos
@@ -91,6 +91,7 @@ for node in args:
     try:
        infoList= vos.getNode(node).getInfoList()
     except:
+       raise
        sys.exit(0)
 
     if not len(infoList)>0:

vmkdir

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 """Create a directory (ContainerNode) in the VOSpace repositotry"""
 
 

vos.py

@@ -12,7 +12,10 @@
 import os
 import errno
 import xml.etree.ElementTree as ET
+BUFSIZE=8192
 
+SERVER="www.cadc.hia.nrc.gc.ca"
+### SERVER="scapa.cadc.dao.nrc.ca"
 
 class urlparse:
     """Break the URL into parts.
@@ -39,13 +42,11 @@ def __str__(self):
 class Connection:
     """Class to hold and act on the X509 certificate"""
 
-    def __init__(self,credServerURL="http://www.cadc.hia.nrc.gc.ca/cred/proxyCert",
-                 certfile=None,save=True,data={'daysValid': 1}):
+    def __init__(self, certfile=None):
         """Setup the Certificate for later usage
 
         cerdServerURL --- the location of the cadc proxy certificate server
         certfile      --- where to store the certificate, if None then ${HOME}/.ssl or a temporary filename
-        save          --- save the certificate for later use? if not then use a tempfilename.
 
         The user must supply a valid certificate. 
         """
@@ -108,7 +109,7 @@ class Node:
     TYPE  ='{%s}type' % XSINS
     NODES ='{%s}nodes' % VOSNS
     NODE  ='{%s}node' % VOSNS
-    PROPERTIES='{%s}properties' % VOSNS
+    PROPERTIES='{%s}properties' % VOSNS 
     PROPERTY='{%s}property' % VOSNS
     ACCEPTS='{%s}accepts' % VOSNS
     PROVIDES='{%s}provides' % VOSNS
@@ -131,6 +132,7 @@ def __init__(self,node,nodeType="vos:DataNode",properties={},xml=None,subnodes=[
         self.props={}
         self.attr={}
         self.xattr={}
+	self._nodeList = None
         self.update()
 
     def update(self):
@@ -143,7 +145,6 @@ def update(self):
             logging.debug(ET.dump(self.node))
             return None
 
-        self.nodeList=self.getNodeList()
         self.uri=self.node.get('uri')
         self.name=os.path.basename(self.uri)
         for propertiesNode in self.node.findall(Node.PROPERTIES):
@@ -201,7 +202,7 @@ def setattr(self,attr={}):
         st_mode=0
         if node.type=='vos:ContainerNode':
             st_mode |= S_IFDIR
-            self.attr['st_nlink']=len(node.nodeList)+2
+            self.attr['st_nlink']=len(node.getNodeList())+2
         else:
             self.attr['st_nlink']=1
             st_mode |= S_IFREG
@@ -343,6 +344,7 @@ def create(self,uri,nodeType="vos:DataNode",properties={},subnodes=[]):
         if not properties.has_key('type'):
             import mimetypes
             properties['type']=mimetypes.guess_type(uri)[0]
+            logging.debug("set type to %s" % (properties['type']))
         propertiesNode=ET.SubElement(node,Node.PROPERTIES)
         for property in properties.keys():
             if not properties[property]==None :
@@ -417,16 +419,22 @@ def getInfo(self):
 
     def getNodeList(self):
         """Get a list of all the nodes held to by a ContainerNode return a list of Node objects"""
-        nodeList=[]
-        for nodesNode in self.node.findall(Node.NODES):
-            for nodeNode in nodesNode.findall(Node.NODE):
-                nodeList.append(Node(nodeNode))
-        return nodeList
+	if (self._nodeList is None):
+	    self._nodeList=[]
+	    for nodesNode in self.node.findall(Node.NODES):
+		for nodeNode in nodesNode.findall(Node.NODE):
+		    self.addChild(nodeNode)
+        return self._nodeList
+
+    def addChild(self,childEt):
+	childNode = Node(childEt)
+	self._nodeList.append(childNode)
+	return(childNode)
 
     def getInfoList(self,longList=True):
         """Retrieve a list of tupples of (NodeName, Info dict)"""
         infoList={}
-        for node in self.nodeList:
+        for node in self.getNodeList():
             infoList[node.name]=node.getInfo()
         if self.type=="vos:DataNode":
             infoList[self.name]=self.getInfo()
@@ -483,6 +491,8 @@ def checkstatus(self):
             if self.resp.status == 404:
                 ### file not found
                 raise IOError(errno.ENOENT,"Node not found",self.url)
+            if self.resp.status == 401:
+	        raise IOError(errno.EACCES,"Not authorized",self.url)
             logging.debug(self.resp.read())
             raise IOError(self.resp.status,"unexpected server response %s (%d)" % ( self.resp.reason, self.resp.status),self.url)
 
@@ -565,10 +575,10 @@ def write(self,buf):
 class Client:
     """The Client object does the work"""
 
-    VOServers={'cadc.nrc.ca!vospace': "www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca",
-               'cadc.nrc.ca~vospace': "www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca"}
+    VOServers={'cadc.nrc.ca!vospace': SERVER,
+               'cadc.nrc.ca~vospace': SERVER}
 
-    VOTransfer='https://www.cadc.hia.nrc.gc.ca/vospace/synctrans'
+    VOTransfer='https://%s/vospace/synctrans' % ( SERVER)
 
     ### reservered vospace properties, not to be used for extended property setting
     vosProperties=["description", "type", "encoding", "MD5", "length", "creator","date",
@@ -598,7 +608,7 @@ def copy(self,src,dest):
 
         totalBytes=0
         while True:
-            buf=fin.read()
+            buf=fin.read(BUFSIZE)
             logging.debug("Read %d bytes from %s" % ( len(buf),src))
             if len(buf)==0:
                 break
@@ -633,19 +643,40 @@ def fixURI(self,uri):
         return "%s://%s/%s" % (parts.scheme, host, path)
 
 
-    def getNode(self,uri):
+    def getNode(self,uri,limit=500):
         """connect to VOSpace and download the definition of vospace node
 
-        target --- a voSpace node in the format vos:/vospaceName/nodeName
+        uri   --- a voSpace node in the format vos:/vospaceName/nodeName
+        limit --- load children nodes in batches of limit
         """
-        xmlObj=self.open(uri,os.O_RDONLY)
+        xmlObj=self.open(uri,os.O_RDONLY, limit=0)
         dom=ET.parse(xmlObj)
         logging.debug("%s" %( str(dom)))
-        return Node(dom.getroot())
+        node = Node(dom.getroot())
+	# If this is a container node and the nodlist has not already been set, try to load the children.
+	# this would be better deferred until the children are actually needed, however that would require
+	# access to a connection when the children are accessed, and thats not easy.
+	if (node.isdir() and len(node.getNodeList()) == 0):
+	    logging.debug("Loading children")
+	    nextURI = None
+	    again = True
+	    while again:
+		again = False
+		getChildrenXMLDoc=self.open(uri,os.O_RDONLY, limit=limit,nextURI=nextURI)
+		getChildrenDOM = ET.parse(getChildrenXMLDoc)
+		for nodesNode in getChildrenDOM.findall(Node.NODES):
+		    for child in nodesNode.findall(Node.NODE):
+			if child.get('uri') != nextURI:
+			    childNode = node.addChild(child)
+			    nextURI = childNode.uri
+			    logging.debug("added child %s" % childNode.uri)
+			    again = True
+	return(node)
 
 
-    def getNodeURL(self,uri,protocol="https", method='GET', view=None):            
+    def getNodeURL(self,uri,protocol="https", method='GET', view=None, limit=0, nextURI=None):            
         """Split apart the node string into parts and return the correct URL for this node"""
+	import urllib
 
         uri   = self.fixURI(uri)
         parts = urlparse(uri)
@@ -660,12 +691,18 @@ def getNodeURL(self,uri,protocol="https", method='GET', view=None):
             return "%s://%s/data/pub/vospace/%s" % (protocol, server,parts.path.strip('/'))
 
         ### this is a GET so we might have to stick some data onto the URL...
+	fields = {'limit': limit}
         if view is not None:
-            import urllib
-            data="?"+urllib.urlencode({'view': view})
-        else:
-            data=''
-        return "%s://%s/vospace/nodes/%s%s" % ( protocol, server, parts.path.strip('/'), data)
+	    fields['view'] = view
+        if nextURI is not None:
+	    fields['uri'] = nextURI
+        logging.debug("method %s " % method)
+	data="?"+urllib.urlencode(fields)
+        logging.debug("method %s " % method)
+	URL = "%s://%s/vospace/nodes/%s%s" % ( protocol, server, parts.path.strip('/'), data)
+        logging.debug("method %s " % method)
+        logging.debug("Accessing URL %s" % URL)
+	return URL
 
     def move(self,srcURI,destURI):
         """Move srcUri to targetUri"""
@@ -685,8 +722,8 @@ def move(self,srcURI,destURI):
         return  False
 
 
-    def open(self, uri, mode=os.O_RDONLY, view=None, head=False, URL=None):
-        """Connect to URL and PUT contents of src to that connection return transfer status"""
+    def open(self, uri, mode=os.O_RDONLY, view=None, head=False, URL=None, limit=0, nextURI=None):
+        """Connect to the uri as a VOFile object"""
 
         # the URL of the connection depends if we are 'getting', 'putting' or 'posting'  data
         method=None
@@ -703,7 +740,7 @@ def open(self, uri, mode=os.O_RDONLY, view=None, head=False, URL=None):
         if not method:
             raise IOError(errno.EOPNOTSUPP,"Invalid access mode", mode)
         if URL is None:
-            URL=self.getNodeURL(uri, method=method, view=view)
+            URL=self.getNodeURL(uri, method=method, view=view,limit=limit,nextURI=nextURI)
         logging.debug(URL)
         return VOFile(URL,self.conn,method=method)
 
@@ -730,7 +767,7 @@ def listdir(self,uri):
         """Walk through the directory structure a al os.walk"""
         logging.debug("getting a listing of %s " % ( uri))
         names=[]
-        for node in self.getNode(uri).nodeList:
+        for node in self.getNode(uri).getNodeList():
             names.append(node.name)
         return names
 

vrm

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 """Create a directory (ContainerNode) in the VOSpace repositotry"""