diff --git a/getCert b/getCert index 695133831..4f11cb586 100644 --- a/getCert +++ b/getCert @@ -65,7 +65,6 @@ parser.add_option('--daysValid', type=int, default=10, help='Number of days the cetificate should be valid (default: 10)') parser.add_option('--dest', default=os.path.join(os.getenv('HOME','/tmp'),".ssl/cadcproxy.pem"), help="Location to store the proxy certifacte") - (opt,args) = parser.parse_args() if __name__=='__main__': diff --git a/mountvofs b/mountvofs index ef8ed5d2e..a984c8b68 100755 --- a/mountvofs +++ b/mountvofs @@ -1,4 +1,4 @@ -#!python +#!/usr/bin/env python """A FUSE based filesystem view of VOSpace.""" from sys import argv, exit, platform @@ -13,6 +13,7 @@ import os import vos from os import O_RDONLY, O_WRONLY, O_RDWR, O_APPEND import logging +READBUF=8192 def flag2mode(flags): md = {O_RDONLY: 'r', O_WRONLY: 'w', O_RDWR: 'w+'} @@ -193,9 +194,8 @@ class VOFS(LoggingMixIn, Operations): node=self.getNode(path) w=self.client.open(node.uri,'w') os.lseek(fh,0,0) - bufsize=2**33 while True: - buf=os.read(fh,bufsize) + buf=os.read(fh,READBUF) logging.debug("Writing %d bytes to %s" % (len(buf), path)) if not buf: break @@ -257,7 +257,7 @@ class VOFS(LoggingMixIn, Operations): self.node[path]=node self.attr[path]=node.attr if self.node[path].isdir(): - for node in self.node[path].nodeList: + for node in self.node[path].getNodeList(): npath=os.path.join(path,node.name) self.node[npath]=node self.attr[npath]=node.attr @@ -270,7 +270,10 @@ class VOFS(LoggingMixIn, Operations): def getattr(self, path, fh=None): """Build some attributes for this file, we have to make-up some stuff""" if not path in self.attr: - node=self.getNode(path) + try: + node=self.client.getNode(path,limit=0) + except IOError as e: + raise FuseOSError(e.errno) self.attr[path]=node.attr logging.debug("Got %s for %s" % (node, path)) atime=self.attr.get('st_atime',time.time()) @@ -280,7 +283,10 @@ class VOFS(LoggingMixIn, Operations): ### the modification/change times are after the last access ### so we should access this VOSpace node again. logging.debug("Getting node details from getattr for path %s" % ( path)) - node=self.getNode(path,force=True) + try: + node=self.client.getNode(path,limit=0) + except IOError as e: + raise FuseOSError(e.errno) self.attr[path]=node.attr #self.attr[path].update(self.getNode(path).attr) return self.attr[path] @@ -370,7 +376,7 @@ class VOFS(LoggingMixIn, Operations): r=self.client.open(path,mode=os.O_RDONLY,view="data") fpos=0 while True: - buf=r.read(2**(10+10)) + buf=r.read(READBUF) logging.debug("Read buffer length %s from %s" % ( len(buf), self.getNode(self.getPath(fh)).name)) if not buf: break @@ -394,7 +400,7 @@ class VOFS(LoggingMixIn, Operations): def readdir(self, path, fh): """Send a list of entried in this directory""" - return ['.','..'] + [e.name.encode('utf-8') for e in self.getNode(path,force=True).nodeList ] + return ['.','..'] + [e.name.encode('utf-8') for e in self.getNode(path,force=True).getNodeList() ] def release(self, path, fh): """Close the file, but if this was a holding spot for writes, then write the file to the node""" @@ -407,7 +413,7 @@ class VOFS(LoggingMixIn, Operations): w=self.client.open(path,os.O_WRONLY) os.lseek(fh,0,0) while True: - buf=os.read(fh,2**(10+10)) + buf=os.read(fh,READBUF) logging.debug("Writing %d bytes to %s" % (len(buf), path)) if not buf: break @@ -515,7 +521,7 @@ class VOFS(LoggingMixIn, Operations): node=self.getNode(path,force=True) #if not node.isdir(): # raise FuseOSError(ENOTDIR) - #if len(node.nodeList)>0: + #if len(node.getNodeList())>0: # raise FuseOSError(ENOTEMPTY) fname=os.path.normpath(self.cache_dir+path) if os.access(fname,os.F_OK): @@ -540,7 +546,7 @@ class VOFS(LoggingMixIn, Operations): sfs['f_blocks']=int(bytes/block_size) sfs['f_bfree']=int(free/block_size) sfs['f_bavail']=int(free/block_size) - sfs['f_files']=len(node.nodeList) + sfs['f_files']=len(node.getNodeList()) sfs['f_ffree']=2*10 sfs['f_favail']=2*10 sfs['f_flags']=0 @@ -557,9 +563,8 @@ class VOFS(LoggingMixIn, Operations): except: raise FuseOSError(EIO) fpos=0 - bufsize=4096 while True: - buf=r.read(bufsize) + buf=r.read(READBUF) if not buf: break chunk=min(length-fpos,bufsize) @@ -630,6 +635,7 @@ if __name__ == "__main__": parser.add_option("--log",action="store",help="File to store debug log to",default="/tmp/vos.err") parser.add_option("--cache_limit",action="store",type=int,help="upper limit on local diskspace to use for file caching",default=50*2**(10+10+10)) parser.add_option("--cache_dir",action="store",help="local directory to use for file caching",default=os.getenv('HOME',default='.')) + parser.add_option("--certfile",help="location of your CADC security certificate file",default=os.path.join(os.getenv("HOME","."),".ssl/cadcproxy.pem")) (opt,args)=parser.parse_args() formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') @@ -643,7 +649,7 @@ if __name__ == "__main__": logging.basicConfig(level=logging.ERROR,format="vofs:%(module)s.%(funcName)s %(message)s",filename=opt.log) logging.debug("Checking connetion to VOSpace ") - conn=vos.Connection() + conn=vos.Connection(certfile=opt.certfile) logging.debug("Got a certificate, connections should work") root = opt.vospace diff --git a/vcp b/vcp index 769f84516..c47406e62 100644 --- a/vcp +++ b/vcp @@ -1,4 +1,4 @@ -#!python +#!/usr/bin/env python """copy files from / to vospace directly without using the FUSE layer""" def isdir(filename): diff --git a/vls b/vls index d7759423b..b7b2d9bee 100644 --- a/vls +++ b/vls @@ -1,4 +1,4 @@ -#!python +#!/usr/bin/env python """vls: list the contents of a voSpace""" import vos @@ -91,6 +91,7 @@ for node in args: try: infoList= vos.getNode(node).getInfoList() except: + raise sys.exit(0) if not len(infoList)>0: diff --git a/vmkdir b/vmkdir index b1e5e3b9c..8ac2aa17a 100644 --- a/vmkdir +++ b/vmkdir @@ -1,4 +1,4 @@ -#!python +#!/usr/bin/env python """Create a directory (ContainerNode) in the VOSpace repositotry""" diff --git a/vos.py b/vos.py index 8a890af97..44b9a5bc8 100644 --- a/vos.py +++ b/vos.py @@ -12,7 +12,10 @@ import os import errno import xml.etree.ElementTree as ET +BUFSIZE=8192 +SERVER="www.cadc.hia.nrc.gc.ca" +### SERVER="scapa.cadc.dao.nrc.ca" class urlparse: """Break the URL into parts. @@ -39,13 +42,11 @@ def __str__(self): class Connection: """Class to hold and act on the X509 certificate""" - def __init__(self,credServerURL="http://www.cadc.hia.nrc.gc.ca/cred/proxyCert", - certfile=None,save=True,data={'daysValid': 1}): + def __init__(self, certfile=None): """Setup the Certificate for later usage cerdServerURL --- the location of the cadc proxy certificate server certfile --- where to store the certificate, if None then ${HOME}/.ssl or a temporary filename - save --- save the certificate for later use? if not then use a tempfilename. The user must supply a valid certificate. """ @@ -108,7 +109,7 @@ class Node: TYPE ='{%s}type' % XSINS NODES ='{%s}nodes' % VOSNS NODE ='{%s}node' % VOSNS - PROPERTIES='{%s}properties' % VOSNS + PROPERTIES='{%s}properties' % VOSNS PROPERTY='{%s}property' % VOSNS ACCEPTS='{%s}accepts' % VOSNS PROVIDES='{%s}provides' % VOSNS @@ -131,6 +132,7 @@ def __init__(self,node,nodeType="vos:DataNode",properties={},xml=None,subnodes=[ self.props={} self.attr={} self.xattr={} + self._nodeList = None self.update() def update(self): @@ -143,7 +145,6 @@ def update(self): logging.debug(ET.dump(self.node)) return None - self.nodeList=self.getNodeList() self.uri=self.node.get('uri') self.name=os.path.basename(self.uri) for propertiesNode in self.node.findall(Node.PROPERTIES): @@ -201,7 +202,7 @@ def setattr(self,attr={}): st_mode=0 if node.type=='vos:ContainerNode': st_mode |= S_IFDIR - self.attr['st_nlink']=len(node.nodeList)+2 + self.attr['st_nlink']=len(node.getNodeList())+2 else: self.attr['st_nlink']=1 st_mode |= S_IFREG @@ -343,6 +344,7 @@ def create(self,uri,nodeType="vos:DataNode",properties={},subnodes=[]): if not properties.has_key('type'): import mimetypes properties['type']=mimetypes.guess_type(uri)[0] + logging.debug("set type to %s" % (properties['type'])) propertiesNode=ET.SubElement(node,Node.PROPERTIES) for property in properties.keys(): if not properties[property]==None : @@ -417,16 +419,22 @@ def getInfo(self): def getNodeList(self): """Get a list of all the nodes held to by a ContainerNode return a list of Node objects""" - nodeList=[] - for nodesNode in self.node.findall(Node.NODES): - for nodeNode in nodesNode.findall(Node.NODE): - nodeList.append(Node(nodeNode)) - return nodeList + if (self._nodeList is None): + self._nodeList=[] + for nodesNode in self.node.findall(Node.NODES): + for nodeNode in nodesNode.findall(Node.NODE): + self.addChild(nodeNode) + return self._nodeList + + def addChild(self,childEt): + childNode = Node(childEt) + self._nodeList.append(childNode) + return(childNode) def getInfoList(self,longList=True): """Retrieve a list of tupples of (NodeName, Info dict)""" infoList={} - for node in self.nodeList: + for node in self.getNodeList(): infoList[node.name]=node.getInfo() if self.type=="vos:DataNode": infoList[self.name]=self.getInfo() @@ -483,6 +491,8 @@ def checkstatus(self): if self.resp.status == 404: ### file not found raise IOError(errno.ENOENT,"Node not found",self.url) + if self.resp.status == 401: + raise IOError(errno.EACCES,"Not authorized",self.url) logging.debug(self.resp.read()) raise IOError(self.resp.status,"unexpected server response %s (%d)" % ( self.resp.reason, self.resp.status),self.url) @@ -565,10 +575,10 @@ def write(self,buf): class Client: """The Client object does the work""" - VOServers={'cadc.nrc.ca!vospace': "www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca", - 'cadc.nrc.ca~vospace': "www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca"} + VOServers={'cadc.nrc.ca!vospace': SERVER, + 'cadc.nrc.ca~vospace': SERVER} - VOTransfer='https://www.cadc.hia.nrc.gc.ca/vospace/synctrans' + VOTransfer='https://%s/vospace/synctrans' % ( SERVER) ### reservered vospace properties, not to be used for extended property setting vosProperties=["description", "type", "encoding", "MD5", "length", "creator","date", @@ -598,7 +608,7 @@ def copy(self,src,dest): totalBytes=0 while True: - buf=fin.read() + buf=fin.read(BUFSIZE) logging.debug("Read %d bytes from %s" % ( len(buf),src)) if len(buf)==0: break @@ -633,19 +643,40 @@ def fixURI(self,uri): return "%s://%s/%s" % (parts.scheme, host, path) - def getNode(self,uri): + def getNode(self,uri,limit=500): """connect to VOSpace and download the definition of vospace node - target --- a voSpace node in the format vos:/vospaceName/nodeName + uri --- a voSpace node in the format vos:/vospaceName/nodeName + limit --- load children nodes in batches of limit """ - xmlObj=self.open(uri,os.O_RDONLY) + xmlObj=self.open(uri,os.O_RDONLY, limit=0) dom=ET.parse(xmlObj) logging.debug("%s" %( str(dom))) - return Node(dom.getroot()) + node = Node(dom.getroot()) + # If this is a container node and the nodlist has not already been set, try to load the children. + # this would be better deferred until the children are actually needed, however that would require + # access to a connection when the children are accessed, and thats not easy. + if (node.isdir() and len(node.getNodeList()) == 0): + logging.debug("Loading children") + nextURI = None + again = True + while again: + again = False + getChildrenXMLDoc=self.open(uri,os.O_RDONLY, limit=limit,nextURI=nextURI) + getChildrenDOM = ET.parse(getChildrenXMLDoc) + for nodesNode in getChildrenDOM.findall(Node.NODES): + for child in nodesNode.findall(Node.NODE): + if child.get('uri') != nextURI: + childNode = node.addChild(child) + nextURI = childNode.uri + logging.debug("added child %s" % childNode.uri) + again = True + return(node) - def getNodeURL(self,uri,protocol="https", method='GET', view=None): + def getNodeURL(self,uri,protocol="https", method='GET', view=None, limit=0, nextURI=None): """Split apart the node string into parts and return the correct URL for this node""" + import urllib uri = self.fixURI(uri) parts = urlparse(uri) @@ -660,12 +691,18 @@ def getNodeURL(self,uri,protocol="https", method='GET', view=None): return "%s://%s/data/pub/vospace/%s" % (protocol, server,parts.path.strip('/')) ### this is a GET so we might have to stick some data onto the URL... + fields = {'limit': limit} if view is not None: - import urllib - data="?"+urllib.urlencode({'view': view}) - else: - data='' - return "%s://%s/vospace/nodes/%s%s" % ( protocol, server, parts.path.strip('/'), data) + fields['view'] = view + if nextURI is not None: + fields['uri'] = nextURI + logging.debug("method %s " % method) + data="?"+urllib.urlencode(fields) + logging.debug("method %s " % method) + URL = "%s://%s/vospace/nodes/%s%s" % ( protocol, server, parts.path.strip('/'), data) + logging.debug("method %s " % method) + logging.debug("Accessing URL %s" % URL) + return URL def move(self,srcURI,destURI): """Move srcUri to targetUri""" @@ -685,8 +722,8 @@ def move(self,srcURI,destURI): return False - def open(self, uri, mode=os.O_RDONLY, view=None, head=False, URL=None): - """Connect to URL and PUT contents of src to that connection return transfer status""" + def open(self, uri, mode=os.O_RDONLY, view=None, head=False, URL=None, limit=0, nextURI=None): + """Connect to the uri as a VOFile object""" # the URL of the connection depends if we are 'getting', 'putting' or 'posting' data method=None @@ -703,7 +740,7 @@ def open(self, uri, mode=os.O_RDONLY, view=None, head=False, URL=None): if not method: raise IOError(errno.EOPNOTSUPP,"Invalid access mode", mode) if URL is None: - URL=self.getNodeURL(uri, method=method, view=view) + URL=self.getNodeURL(uri, method=method, view=view,limit=limit,nextURI=nextURI) logging.debug(URL) return VOFile(URL,self.conn,method=method) @@ -730,7 +767,7 @@ def listdir(self,uri): """Walk through the directory structure a al os.walk""" logging.debug("getting a listing of %s " % ( uri)) names=[] - for node in self.getNode(uri).nodeList: + for node in self.getNode(uri).getNodeList(): names.append(node.name) return names diff --git a/vrm b/vrm index 4578835a9..475d171f8 100644 --- a/vrm +++ b/vrm @@ -1,4 +1,4 @@ -#!python +#!/usr/bin/env python """Create a directory (ContainerNode) in the VOSpace repositotry""" diff --git a/vrmdir b/vrmdir index f360610ea..0edb53d8d 100644 --- a/vrmdir +++ b/vrmdir @@ -1,4 +1,4 @@ -#!python +#!/usr/bin/env python """Create a directory (ContainerNode) in the VOSpace repositotry"""