Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 956 lines (807 sloc) 35.974 kb
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
1 """A set of Python Classes for connecting to and interacting with a VOSpace service.
2
3 Connections to VOSpace are made using a SSL X509 certificat which is stored in a .pem file.
4 The certificate is supplied by the user or by the CADC credential server
5
6 """
7
8 import logging
9 import time
10 import threading
11 import sys
12 import os
13 import errno
14 import xml.etree.ElementTree as ET
2d559e0 @ijiraq special handelling of 409 error codes
ijiraq authored
15 import re
151b816 Added user-agent tag and corrected error logging for vls
JJ Kavelaars authored
16 from __version__ import version
d68492d @igable Increase the buffer size to 1 MB and removing the loggin from the
igable authored
17 # set a 1 MB buffer to keep the number of trips
18 # around the IO loop small
19
20 BUFSIZE=8388608
21 #BUFSIZE=8192
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
22
030c36b @ijiraq merge with SVN tree
ijiraq authored
23 SERVER="www.cadc.hia.nrc.gc.ca"
24 ### SERVER="scapa.cadc.dao.nrc.ca"
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
25
26 class urlparse:
27 """Break the URL into parts.
28
29 There is a difference between the 2.5 and 2.7 version of the urlparse.urlparse command, so here I roll my own..."""
30
31 def __init__(self,url):
32 import re
33
ea35908 Not really sure
JJ Kavelaars authored
34 m=re.match("(^(?P<scheme>[a-zA-Z]*):)?(//(?P<netloc>[^/]*))?(?P<path>/?.*)?",url)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
35 if not m.group:
36 return None
37 self.scheme=m.group('scheme')
38 self.netloc=m.group('netloc')
39 self.path=m.group('path')
40
41 def __str__(self):
ea35908 Not really sure
JJ Kavelaars authored
42 #return "[scheme: %s, netloc: %s, path: %s, frag: %s, query: %s]" % ( self.scheme, self.netloc, self.path,self.frag,self.query)
43 return "[scheme: %s, netloc: %s, path: %s]" % ( self.scheme, self.netloc, self.path)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
44
45
46
47 class Connection:
48 """Class to hold and act on the X509 certificate"""
49
030c36b @ijiraq merge with SVN tree
ijiraq authored
50 def __init__(self, certfile=None):
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
51 """Setup the Certificate for later usage
52
53 cerdServerURL --- the location of the cadc proxy certificate server
54 certfile --- where to store the certificate, if None then ${HOME}/.ssl or a temporary filename
55
fb88fe5 Pulled the getCert method and dependencies out... getCert now stand a…
JJ Kavelaars authored
56 The user must supply a valid certificate.
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
57 """
58
59 ## figure out a filename and open that file for writing
60 if not certfile:
61 ## No filename see if we can find a HOME directory
62 dirName=os.getenv('HOME')
63 logging.debug("looking for certificate in %s" % ( dirName))
64 if not dirName:
2bc921e Some bug fixxes and moved back to optparse to stay P2.6 compatible
JJ Kavelaars authored
65 raise IOError(errno.EEXIST,"HOME is not defined for your environment")
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
66 certDir=os.path.join(dirName,'.ssl')
67 logging.debug("looking for certificate in %s" % ( certDir))
68 if not os.access(certDir,os.F_OK):
69 os.mkdir(certDir)
70 certfile = os.path.join(certDir,"cadcproxy.pem")
71 logging.debug("looking for certificate in %s" % ( certfile))
72 if not os.access(certfile,os.F_OK):
ecfaa86 @ijiraq Added 'try/except' to trap failures when vos client fails to be creat…
ijiraq authored
73 raise EnvironmentError(errno.EACCES,"No certifacte file found at %s\n (Perhaps use getCert to pull one)" %(certfile))
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
74
75 logging.debug("requesting password")
76
77 self.certfile=certfile
78
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
79 logging.debug("Using certificate file %s" % (self.certfile))
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
80
81
82 def getConnection(self,url):
83 """Create an HTTPSConnection object and return. Uses the client certificate if None given.
84
85 uri -- a VOSpace uri (vos://cadc.nrc.ca~vospace/path)
86 certFilename -- the name of the certificate pem file.
87 """
88 parts=urlparse(url)
89 ports={"http": 80, "https": 443}
90 certfile=self.certfile
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
91 #logging.debug("Trying to connect to %s://%s using %s" % (parts.scheme,parts.netloc,certfile))
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
92
ecfaa86 @ijiraq Added 'try/except' to trap failures when vos client fails to be creat…
ijiraq authored
93 import httplib, ssl
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
94 try:
95 if parts.scheme=="https":
96 connection = httplib.HTTPSConnection(parts.netloc,key_file=certfile,cert_file=certfile,timeout=600)
97 else:
98 connection = httplib.HTTPConnection(parts.netloc,timeout=600)
3bc0069 @ijiraq improved error logging
ijiraq authored
99 except httplib.NotConnected as e:
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
100 logging.error("HTTP connection to %s failed \n" % (parts.netloc))
3bc0069 @ijiraq improved error logging
ijiraq authored
101 logging.error("%s \n" % ( str(e)))
1639cdd @ijiraq Added better Error handling so that VCP error trapping re-Raises erro…
ijiraq authored
102 raise OSError(errno.ENTCONN,"VOSpace connection failed",parts.netloc)
af3e34a Moved the loop until connection succeeds into the connection object c…
JJ Kavelaars authored
103
104
105 ## Try to open this connection.
106 timestart = time.time()
107 while True:
108 try:
109 connection.connect()
110 except httplib.HTTPException as e:
111 logging.critical("%s" % ( str(e)))
112 logggin.critical("Retrying connection for 1200 seconds")
113 if time.time() - timestart > 1200:
114 raise e
4cd0431 Improved error passing so certificate failures more obvious
JJ Kavelaars authored
115 except Exception as e:
116 logging.error(str(e))
117 logging.error("Perhaps your proxy certificate is expired?")
118 ex=IOError()
119 ex.errno=errno.ECONNREFUSED
120 ex.strerror="VOSpace connection failed"
121 ex.filename=parts.netloc
122 raise ex
af3e34a Moved the loop until connection succeeds into the connection object c…
JJ Kavelaars authored
123 break
124
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
125 #logging.debug("Returning connection " )
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
126 return connection
127
128
129
130 class Node:
131 """A VOSpace node"""
132
133 IVOAURL="ivo://ivoa.net/vospace/core"
134 CADCURL="ivo://cadc.nrc.ca/vospace/core"
135
136 VOSNS="http://www.ivoa.net/xml/VOSpace/v2.0"
137 XSINS="http://www.w3.org/2001/XMLSchema-instance"
138 TYPE ='{%s}type' % XSINS
139 NODES ='{%s}nodes' % VOSNS
140 NODE ='{%s}node' % VOSNS
030c36b @ijiraq merge with SVN tree
ijiraq authored
141 PROPERTIES='{%s}properties' % VOSNS
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
142 PROPERTY='{%s}property' % VOSNS
143 ACCEPTS='{%s}accepts' % VOSNS
144 PROVIDES='{%s}provides' % VOSNS
145
146 def __init__(self,node,nodeType="vos:DataNode",properties={},xml=None,subnodes=[]):
147 """Create a Node object based on the DOM passed to the init method
148
149 if node is a string then create a node named node of nodeType with properties
150 """
151
152 if type(node)==str:
153 node=self.create(node,nodeType,properties,subnodes=subnodes)
154
155 if node is None:
156 raise LookupError("no node found or created?")
157
158 self.node=node
159 self.node.set('xmlns:vos',self.VOSNS)
160 self.type=None
161 self.props={}
162 self.attr={}
db935db Added some basic xattr support
JJ Kavelaars authored
163 self.xattr={}
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
164 self._nodeList = None
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
165 self.update()
166
167 def update(self):
168 """Update the convience links of this node as we update the xml file"""
169
170 self.type=self.node.get(Node.TYPE)
171 if self.type == None:
2bc921e Some bug fixxes and moved back to optparse to stay P2.6 compatible
JJ Kavelaars authored
172 logging.debug("Node type unknown, no node created")
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
173 import xml.etree.ElementTree as ET
174 logging.debug(ET.dump(self.node))
175 return None
176
177 self.uri=self.node.get('uri')
178 self.name=os.path.basename(self.uri)
179 for propertiesNode in self.node.findall(Node.PROPERTIES):
180 self.setProps(propertiesNode)
181 self.isPublic=False
182 if self.props.get('ispublic','false')=='true':
183 self.isPublic=True
ca5a55e stop using 'NONE' where NULL expected
JJ Kavelaars authored
184 self.groupwrite = self.props.get('groupwrite','')
185 self.groupread = self.props.get('groupread','')
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
186 self.setattr()
db935db Added some basic xattr support
JJ Kavelaars authored
187 self.setxattr()
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
188
b755f98 Added new function to push only 'changed' or 'deleted' properties to …
JJ Kavelaars authored
189 def setProperty(self,key,value):
190 """Given a dictionary of props build a properies subelement"""
191 properties=self.node.find(Node.PROPERTIES)
192 uri="%s#%s" %(Node.IVOAURL,key)
193 ET.SubElement(properties,Node.PROPERTY,
194 attrib={'uri': uri,'readOnly': 'false'}).text=value
195
196
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
197 def __str__(self):
198 import xml.etree.ElementTree as ET
199 class dummy:
200 pass
201 data=[]
202 file=dummy()
203 file.write=data.append
204 ET.ElementTree(self.node).write(file,encoding="UTF-8")
205 return "".join(data)
206
207 def setattr(self,attr={}):
208 """return a dictionary of attributes associated with the file stored at node
209
210 These attributes are determind from the node on VOSpace.
211 """
212 ## Get the flags for file mode settings.
213 from stat import S_IFDIR, S_IFREG
214 from stat import S_IRUSR, S_IRGRP, S_IROTH
215 from stat import S_IWUSR, S_IWGRP, S_IWOTH
216 from stat import S_IXUSR, S_IXGRP, S_IXOTH
217 from os import getgid, getuid
218
219 self.attr={}
220 node=self
221
222 ## Only one date provided by VOSpace, so use this as all possible dates.
223 sdate = node.props.get('date',None)
224 atime=time.time()
225 if not sdate:
226 mtime = atime
227 else:
228 ### mktime is expecting a localtime but we're sending a UT date, so some correction will be needed
229 mtime=time.mktime(time.strptime(sdate[0:-4],'%Y-%m-%dT%H:%M:%S'))
ca0ab76 @ijiraq Corrected various reference errors (attr[path]) and some rules about …
ijiraq authored
230 mtime=mtime - time.mktime(time.gmtime()) + time.mktime(time.localtime())
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
231 self.attr['st_ctime']=attr.get('st_ctime',mtime)
232 self.attr['st_mtime']=attr.get('st_mtime',mtime)
233 self.attr['st_atime']=atime
234
235 ## set the MODE by orring together all flags from stat
236 st_mode=0
237 if node.type=='vos:ContainerNode':
238 st_mode |= S_IFDIR
030c36b @ijiraq merge with SVN tree
ijiraq authored
239 self.attr['st_nlink']=len(node.getNodeList())+2
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
240 else:
241 self.attr['st_nlink']=1
242 st_mode |= S_IFREG
243
244 ## Set the OWNER permissions
245 ## All files are read/write/execute by owner...
246 st_mode |= S_IRUSR | S_IWUSR | S_IXUSR
247
248 ## Set the GROUP permissions
249 if node.props.get('groupwrite',"NONE")!="NONE":
250 st_mode |= S_IWGRP
251 if node.props.get('groupread',"NONE")!="NONE":
252 st_mode |= S_IRGRP
253 st_mode |= S_IXGRP
254
255 ## Set the OTHER permissions
256 if node.props.get('ispublic','false')=='true':
257 ## If you can read the file then you can execute too.
258 ## Public does NOT mean writeable. EVER
259 st_mode |= S_IROTH | S_IXOTH
260
261 self.attr['st_mode']=attr.get('st_mode',st_mode)
262
263 ## We set the owner and group bits to be those of the currently running process.
264 ## This is a hack since we don't have an easy way to figure these out. TBD!
265 self.attr['st_uid']=attr.get('st_uid',getuid())
266 self.attr['st_gid']=attr.get('st_uid',getgid())
267 self.attr['st_size']=attr.get('st_size',int(node.props.get('length',0)))
268
db935db Added some basic xattr support
JJ Kavelaars authored
269 def setxattr(self, attrs={}):
270 """Initialize the attributes using the properties sent with the node"""
271 for key in self.props:
272 if key in Client.vosProperties:
273 continue
274 self.xattr[key]=self.props[key]
275 return
276
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
277 def chwgrp(self,group):
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
278 """Set the groupwrite value for this node"""
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
279 self.groupwrite=group
280 return self.changeProp('groupwrite',group)
281
282 def chrgrp(self,group):
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
283 """Set the groupread value for this node"""
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
284 self.groupread=group
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
285 return self.changeProp('groupread', group)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
286
287 def setPublic(self,value):
288 logging.debug("Setting value of ispublic to %s" % (str(value)))
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
289 return self.changeProp('ispublic', value)
b755f98 Added new function to push only 'changed' or 'deleted' properties to …
JJ Kavelaars authored
290
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
291
292 def changeProp(self,key,value):
b755f98 Added new function to push only 'changed' or 'deleted' properties to …
JJ Kavelaars authored
293 """Change the node property 'key' to 'value'.
294
295 Return 1 if changed.
296
297 This function should be split into 'set' and 'delete'
298 """
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
299 import urllib
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
300 #logging.debug("Before change node XML\n %s" % ( self))
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
301 changed=0
b755f98 Added new function to push only 'changed' or 'deleted' properties to …
JJ Kavelaars authored
302 found=False
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
303 properties = self.node.findall(Node.PROPERTIES)
304 for props in properties:
305 for prop in props.findall(Node.PROPERTY):
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
306 uri=prop.attrib.get('uri',None)
307 propName=urllib.splittag(uri)[1]
308 if propName != key:
db935db Added some basic xattr support
JJ Kavelaars authored
309 continue
b755f98 Added new function to push only 'changed' or 'deleted' properties to …
JJ Kavelaars authored
310 found=True
311 if value is None:
312 ## this is actually a delete property
313 prop.attrib['xsi:nil']='true'
4b6c3be @ijiraq added xmlns:xsi to changeProp
ijiraq authored
314 prop.attrib["xmlns:xsi"]=Node.XSINS
b755f98 Added new function to push only 'changed' or 'deleted' properties to …
JJ Kavelaars authored
315 prop.text = ""
316 self.props[propName]=None
317 else:
318 prop.text=value
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
319 changed=1
b755f98 Added new function to push only 'changed' or 'deleted' properties to …
JJ Kavelaars authored
320 if found or value is None:
321 return changed
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
322 ### must not have had this kind of property already, so set value
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
323 #logging.debug("Adding a property: %s" %(key))
f0108d2 Small tweeks to improve logic
JJ Kavelaars authored
324 propertyNode=ET.SubElement(props,Node.PROPERTY)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
325 propertyNode.attrib['readOnly']="false"
326 ### There should be a '#' in there someplace...
327 propertyNode.attrib["uri"]="%s#%s" % (Node.IVOAURL,key)
328 propertyNode.text=value
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
329 #logging.debug("After change node XML\n %s" %( self))
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
330 return 1
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
331
332
333 def chmod(self,mode):
334 """Set the MODE of this Node...
335
336 translates unix style MODE to voSpace and updates the properties...
337
338 This function is quite limited. We can make a file publicly
339 readable and we can set turn off group read/write permissions,
340 that's all. """
341
342 import urllib
343
344 from stat import S_IFDIR, S_IFREG
345 from stat import S_IRUSR, S_IRGRP, S_IROTH
346 from stat import S_IWUSR, S_IWGRP, S_IWOTH
347 from stat import S_IXUSR, S_IXGRP, S_IXOTH
348 changed=0
349
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
350 #logging.debug("Changing mode to %d" % ( mode))
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
351 if mode & (S_IROTH ) :
352 changed += self.setPublic('true')
353 else:
354 changed += self.setPublic('false')
355
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
356 if mode & (S_IRGRP ):
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
357
358 changed += self.chrgrp(self.groupread)
359 else:
ca5a55e stop using 'NONE' where NULL expected
JJ Kavelaars authored
360 changed += self.chrgrp('')
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
361
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
362 if mode & S_IWGRP :
363 changed += self.chwgrp(self.groupwrite)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
364 else:
ca5a55e stop using 'NONE' where NULL expected
JJ Kavelaars authored
365 changed += self.chwgrp('')
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
366
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
367 #logging.debug("%d -> %s" % ( changed, changed>0))
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
368 return changed>0
369
370
371 def create(self,uri,nodeType="vos:DataNode",properties={},subnodes=[]):
372 """Build the XML needed to represent a VOSpace node returns an ElementTree represenation of the XML
373
374 nodeType -- the VOSpace node type, likely one of vos:DataNode or vos:ContainerNode
375 properties -- a dictionary of the node properties, all assumed to be single words from the IVOA list
376 """
377
378 import xml.etree.ElementTree as ET
379
380 ### Build the root node called 'node'
381 node=ET.Element("node")
382 node.attrib["xmlns"]=Node.VOSNS
383 node.attrib["xmlns:vos"]=Node.VOSNS
384 node.attrib[Node.TYPE]=nodeType
385 node.attrib["busy"]="false"
386 node.attrib["uri"]=uri
387
388 ### create a properties section
389 if not properties.has_key('type'):
390 import mimetypes
391 properties['type']=mimetypes.guess_type(uri)[0]
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
392 #logging.debug("set type to %s" % (properties['type']))
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
393 propertiesNode=ET.SubElement(node,Node.PROPERTIES)
394 for property in properties.keys():
395 if not properties[property]==None :
396 propertyNode=ET.SubElement(propertiesNode,Node.PROPERTY)
397 propertyNode.attrib['readOnly']="false"
398 ### There should be a '#' in there someplace...
399 propertyNode.attrib["uri"]="%s#%s" % (Node.IVOAURL,property)
400 if len(properties[property])>0:
401 propertyNode.text=properties[property]
402
403 ### create accepts
404 accepts=ET.SubElement(node,Node.ACCEPTS)
405
406 ET.SubElement(accepts,"view").attrib['uri']="%s#%s" % (Node.IVOAURL,"defaultview")
407
408 ### create provides section
409 provides=ET.SubElement(node,Node.PROVIDES)
410 ET.SubElement(provides,"view").attrib['uri']="%s#%s" % (Node.IVOAURL,'defaultview')
411 ET.SubElement(provides,"view").attrib['uri']="%s#%s" % (Node.CADCURL,'rssview')
412
413 ### Only DataNode can have a dataview...
414 if nodeType=="vos:DataNode":
415 ET.SubElement(provides,"view").attrib['uri']="%s#%s" % (Node.CADCURL,'dataview')
416
417 ### if this is a container node then we need to add an empy directory contents area...
418 if nodeType=="vos:ContainerNode":
419 nodeList=ET.SubElement(node,Node.NODES)
420 for subnode in subnodes:
421 nodeList.append(subnode.node)
422 #logging.debug(ET.tostring(node,encoding="UTF-8"))
423
424 return node
425
426 def isdir(self):
427 """Check if target is a container Node"""
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
428 #logging.debug(self.type)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
429 if self.type=="vos:ContainerNode":
430 return True
431 return False
432
433 def getInfo(self):
434 """Organize some information about a node and return as dictionary"""
435 import re,time,string,math,urllib
436 date=time.mktime(time.strptime(self.props['date'][0:-4],'%Y-%m-%dT%H:%M:%S'))
437 #if date.tm_year==time.localtime().tm_year:
438 # dateString=time.strftime('%d %b %H:%S',date)
439 #else:
440 # dateString=time.strftime('%d %b %Y',date)
c53ca73 added a drop-through of unknown when creator is not set
JJ Kavelaars authored
441 creator=string.lower(re.search('CN=([^,]*)',self.props.get('creator','CN=unknown_000,')).groups()[0].replace(' ','_'))
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
442 perm=[]
443 writeGroup=""
444 readGroup=""
445 for i in range(10):
446 perm.append('-')
447 perm[1]='r'
448 perm[2]='w'
449 if self.type=="vos:ContainerNode":
450 perm[0]='d'
451 if self.props.get('ispublic',"false")=="true":
452 perm[-3]='r'
453 writeGroup = self.props.get('groupwrite','NONE')
454 if writeGroup != 'NONE':
455 perm[5]='w'
ca5a55e stop using 'NONE' where NULL expected
JJ Kavelaars authored
456 readGroup = self.props.get('groupread','NONE')
457 if readGroup != 'NONE':
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
458 perm[4]='r'
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
459 #logging.debug("%s: %s" %( self.name,self.props))
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
460 return {"permisions": string.join(perm,''),
461 "creator": creator,
462 "readGroup": readGroup,
463 "writeGroup": writeGroup,
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
464 "size": float(self.props.get('length',0)),
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
465 "date": date}
466
467 def getNodeList(self):
468 """Get a list of all the nodes held to by a ContainerNode return a list of Node objects"""
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
469 if (self._nodeList is None):
470 self._nodeList=[]
471 for nodesNode in self.node.findall(Node.NODES):
472 for nodeNode in nodesNode.findall(Node.NODE):
473 self.addChild(nodeNode)
030c36b @ijiraq merge with SVN tree
ijiraq authored
474 return self._nodeList
475
476 def addChild(self,childEt):
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
477 childNode = Node(childEt)
478 self._nodeList.append(childNode)
479 return(childNode)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
480
481 def getInfoList(self,longList=True):
482 """Retrieve a list of tupples of (NodeName, Info dict)"""
483 infoList={}
030c36b @ijiraq merge with SVN tree
ijiraq authored
484 for node in self.getNodeList():
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
485 infoList[node.name]=node.getInfo()
486 if self.type=="vos:DataNode":
487 infoList[self.name]=self.getInfo()
488 return infoList.items()
489
490 def setProps(self,props):
491 """Set the properties of node, given the properties element of that node"""
492 for propertyNode in props.findall(Node.PROPERTY):
493 self.props[self.getPropName(propertyNode)]=self.getPropValue(propertyNode)
494 return
495
496
497 def getPropName(self,prop):
498 """parse the property uri and get the name of the property"""
499 import urllib
500 (url,propName)=urllib.splittag(prop.get('uri'))
501 return propName
502
503 def getPropValue(self,prop):
504 """Pull out the value part of node"""
505 return prop.text
506
507
508 class VOFile:
509 """A class for managing http connecctions"""
510
5ba5b61 Modified vos.py to allow 'size' of file to be sent, used in vsync che…
JJ Kavelaars authored
511 def __init__(self,URL,connector,method,size=None):
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
512 self.closed=True
513 self.resp=503
514 self.connector=connector
515 self.httpCon=None
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
516 self.timeout=-1
517 self.size=size
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
518 self.name=os.path.basename(URL)
519 self._fpos=0
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
520 self.open(URL,method)
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
521
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
522 def tell(self):
523 return self._fpos
524
525 def seek(self,offset,loc=os.SEEK_SET):
526 if loc == os.SEEK_CUR:
527 self._fpos += offset
528 elif loc == os.SEEK_SET:
529 self._fpos = offset
530 elif loc == os.SEEK_END:
531 self._fpos = self.size-offset
532 return
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
533
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
534 def close(self,code=(200, 201, 202, 206, 302, 303, 503)):
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
535 """close the connection"""
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
536 #logging.debug("inside the close")
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
537 if self.closed:
538 return
539 logging.debug("Closing connection")
3bc0069 @ijiraq improved error logging
ijiraq authored
540 try:
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
541 self.httpCon.send('0\r\n\r\n')
542 self.resp=self.httpCon.getresponse()
543 self.httpCon.close()
3bc0069 @ijiraq improved error logging
ijiraq authored
544 except Exception as e:
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
545 logging.error("%s \n" % str(e))
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
546 self.closed=True
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
547 logging.debug("Connection closed")
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
548 return self.checkstatus(codes=code)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
549
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
550 def checkstatus(self, codes=(200, 201, 202, 206, 302, 303, 503, 416)):
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
551 """check the response status"""
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
552 logging.debug("status %d for URL %s" % ( self.resp.status,self.url))
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
553 if self.resp.status not in codes:
2bc921e Some bug fixxes and moved back to optparse to stay P2.6 compatible
JJ Kavelaars authored
554 if self.resp.status == 404:
555 ### file not found
556 raise IOError(errno.ENOENT,"Node not found",self.url)
030c36b @ijiraq merge with SVN tree
ijiraq authored
557 if self.resp.status == 401:
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
558 raise IOError(errno.EACCES,"Not authorized",self.url)
2d559e0 @ijiraq special handelling of 409 error codes
ijiraq authored
559 if self.resp.status == 409:
560 if re.search("DuplicateNode",self.resp.read()):
561 raise OSError(errno.EEXIST,"File Exists",self.url)
e4745f5 cleaned up error code reports
JJ Kavelaars authored
562 msg=self.resp.read()
563 logging.debug(msg)
564 raise IOError(self.resp.status,msg,self.url)
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
565 self.size=self.resp.getheader("Content-Length",0)
af3e34a Moved the loop until connection succeeds into the connection object c…
JJ Kavelaars authored
566 return True
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
567
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
568 def open(self,URL,method="GET", bytes=None):
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
569 """Open a connection to the given URL"""
af3e34a Moved the loop until connection succeeds into the connection object c…
JJ Kavelaars authored
570 import ssl,httplib, sys, mimetypes
c281e83 cleaned up some of the error messages to be clearer about what action…
JJ Kavelaars authored
571 logging.debug("Opening %s (%s)" % (URL, method))
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
572 self.url=URL
573 self.httpCon = self.connector.getConnection(URL)
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
574 #self.httpCon.set_debuglevel(2)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
575 self.closed=False
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
576 #logging.debug("putting request")
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
577 self.httpCon.putrequest(method,URL)
8fcdad8 @ijiraq Remove error messaging on 503 with no timeout provided
ijiraq authored
578 userAgent='vos '+version
579 if "mountvofs" in sys.argv[0]:
580 userAgent='vofs '+version
581 self.httpCon.putheader("User-Agent", userAgent)
2bc921e Some bug fixxes and moved back to optparse to stay P2.6 compatible
JJ Kavelaars authored
582 if method in ["PUT", "POST", "DELETE"]:
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
583 if self.size is not None and type(self.size)==int:
584 self.httpCon.putheader("Content-Length",self.size)
2bc921e Some bug fixxes and moved back to optparse to stay P2.6 compatible
JJ Kavelaars authored
585 contentType="text/xml"
586 if method=="PUT":
587 contentType=mimetypes.guess_type(URL)[0]
588 self.httpCon.putheader("Content-Type",contentType)
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
589 if bytes is not None and method=="GET" :
590 logging.debug("Range: %s" %(bytes))
591 self.httpCon.putheader("Range",bytes)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
592 self.httpCon.putheader("Transfer-Encoding",'chunked')
593 self.httpCon.putheader("Accept", "*/*")
594 self.httpCon.endheaders()
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
595 #logging.debug("Done setting headers")
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
596
597
598 def read(self,size=None):
599 """return size bytes from the connection response"""
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
600 #logging.debug("Starting to read file by closing http(s) connection")
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
601 if not self.closed:
602 self.close()
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
603 bytes=None
604 #if size != None:
605 # bytes = "bytes=%d-" % ( self._fpos)
606 # bytes = "%s%d" % (bytes,self._fpos+size)
607 #self.open(self.url,bytes=bytes,method="GET")
608 #self.close(code=[200,206,303,302,503,404,416])
609 if self.resp.status == 416:
610 return ""
7f85983 @igable change ordering of response code checks
igable authored
611 # check the most likely response first
612 if self.resp.status == 200:
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
613 buff=self.resp.read(size)
614 return buff
615 if self.resp.status == 206:
616 buff=self.resp.read(size)
617 self._fpos += len(buff)
618 logging.debug("left file pointer at: %d" %(self._fpos))
619 return buff
7f85983 @igable change ordering of response code checks
igable authored
620 elif self.resp.status == 404:
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
621 raise IOError(errno.ENFILE)
7f85983 @igable change ordering of response code checks
igable authored
622 elif self.resp.status == 303 or self.resp.status == 302:
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
623 URL = self.resp.getheader('Location',None)
624 if not URL:
2bc921e Some bug fixxes and moved back to optparse to stay P2.6 compatible
JJ Kavelaars authored
625 raise IOError(errno.ENOENT,"No Location on redirect",self.url)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
626 self.open(URL,"GET")
627 return self.read(size)
7f85983 @igable change ordering of response code checks
igable authored
628 elif self.resp.status == 503:
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
629 ## try again in Retry-After seconds or fail
8fcdad8 @ijiraq Remove error messaging on 503 with no timeout provided
ijiraq authored
630 logging.error("Got 503: server busy on %s" % (self.url))
631 try:
632 ras=int(self.resp.getheader("Retry-After",5))
633 except:
634 ras=5
635 logging.error("retrying in %d seconds" % (ras))
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
636 time.sleep(int(ras))
637 self.open(self.url,"GET")
638 return self.read(size)
7f85983 @igable change ordering of response code checks
igable authored
639 # line below can be removed after we are sure all codes
640 # are caught
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
641 raise IOError(self.resp.status,"unexpected server response %s (%d)" % ( self.resp.reason, self.resp.status),self.url)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
642
643
644 def write(self,buf):
645 """write buffer to the connection"""
646 if not self.httpCon or self.closed:
1639cdd @ijiraq Added better Error handling so that VCP error trapping re-Raises erro…
ijiraq authored
647 raise OSError(errno.ENOTCONN,"no connection for write",self.url)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
648 self.httpCon.send('%X\r\n' % len(buf))
649 self.httpCon.send(buf+'\r\n')
650 return len(buf)
651
652
653 class Client:
654 """The Client object does the work"""
655
030c36b @ijiraq merge with SVN tree
ijiraq authored
656 VOServers={'cadc.nrc.ca!vospace': SERVER,
657 'cadc.nrc.ca~vospace': SERVER}
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
658
030c36b @ijiraq merge with SVN tree
ijiraq authored
659 VOTransfer='https://%s/vospace/synctrans' % ( SERVER)
9ec6a01 @ijiraq added vos to vos rename
ijiraq authored
660
db935db Added some basic xattr support
JJ Kavelaars authored
661 ### reservered vospace properties, not to be used for extended property setting
662 vosProperties=["description", "type", "encoding", "MD5", "length", "creator","date",
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
663 "groupread", "groupwrite", "ispublic"]
664
665
666 def __init__(self,certFile=os.path.join(os.getenv('HOME'),'.ssl/cadcproxy.pem'),
ce41d51 @ijiraq Made the client more archive agnostic
ijiraq authored
667 rootNode=None,conn=None,archive='vospace'):
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
668 """This could/should be expanded to set various defaults"""
669 if not conn:
fb88fe5 Pulled the getCert method and dependencies out... getCert now stand a…
JJ Kavelaars authored
670 conn=Connection(certfile=certFile)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
671 self.conn=conn
672 self.VOSpaceServer="cadc.nrc.ca!vospace"
673 self.rootNode=rootNode
ce41d51 @ijiraq Made the client more archive agnostic
ijiraq authored
674 self.archive=archive
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
675 return
676
5ba5b61 Modified vos.py to allow 'size' of file to be sent, used in vsync che…
JJ Kavelaars authored
677 def copy(self,src,dest,sendMD5=False):
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
678 """copy to/from vospace"""
5ba5b61 Modified vos.py to allow 'size' of file to be sent, used in vsync che…
JJ Kavelaars authored
679 import os,hashlib
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
680
18be4df @ijiraq added MD5 checking on copy to/from vospace
ijiraq authored
681 checkSource=False
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
682 if src[0:4]=="vos:":
c281e83 cleaned up some of the error messages to be clearer about what action…
JJ Kavelaars authored
683 srcSize=self.getNode(src).attr['st_size']
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
684 fin=self.open(src,os.O_RDONLY,view='data')
685 fout=open(dest,'w')
18be4df @ijiraq added MD5 checking on copy to/from vospace
ijiraq authored
686 checkSource=True
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
687 else:
c281e83 cleaned up some of the error messages to be clearer about what action…
JJ Kavelaars authored
688 srcSize=os.lstat(src).st_size
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
689 fin=open(src,'r')
c281e83 cleaned up some of the error messages to be clearer about what action…
JJ Kavelaars authored
690 fout=self.open(dest,os.O_WRONLY,size=srcSize)
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
691
c281e83 cleaned up some of the error messages to be clearer about what action…
JJ Kavelaars authored
692 destSize=0
5ba5b61 Modified vos.py to allow 'size' of file to be sent, used in vsync che…
JJ Kavelaars authored
693 md5=hashlib.md5()
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
694 while True:
030c36b @ijiraq merge with SVN tree
ijiraq authored
695 buf=fin.read(BUFSIZE)
d68492d @igable Increase the buffer size to 1 MB and removing the loggin from the
igable authored
696 # In this tight loop I don't think you want logging
697 #logging.debug("Read %d bytes from %s" % ( len(buf),src))
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
698 if len(buf)==0:
699 break
700 fout.write(buf)
5ba5b61 Modified vos.py to allow 'size' of file to be sent, used in vsync che…
JJ Kavelaars authored
701 md5.update(buf)
c281e83 cleaned up some of the error messages to be clearer about what action…
JJ Kavelaars authored
702 destSize+=len(buf)
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
703 fout.close()
704 fin.close()
18be4df @ijiraq added MD5 checking on copy to/from vospace
ijiraq authored
705
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
706 if checkSource:
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
707 checkMD5=self.getNode(src).props.get('MD5','d41d8cd98f00b204e9800998ecf8427e')
18be4df @ijiraq added MD5 checking on copy to/from vospace
ijiraq authored
708 else:
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
709 checkMD5=self.getNode(dest).props.get('MD5','d41d8cd98f00b204e9800998ecf8427e')
18be4df @ijiraq added MD5 checking on copy to/from vospace
ijiraq authored
710
5ba5b61 Modified vos.py to allow 'size' of file to be sent, used in vsync che…
JJ Kavelaars authored
711 if sendMD5:
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
712 if checkMD5 != md5.hexdigest():
e6495ea @ijiraq send a better error message
ijiraq authored
713 logging.error("MD5s don't match ( %s -> %s ) " % ( src, dest))
1639cdd @ijiraq Added better Error handling so that VCP error trapping re-Raises erro…
ijiraq authored
714 raise OSError(errno.EIO,"MD5s don't match",src)
5ba5b61 Modified vos.py to allow 'size' of file to be sent, used in vsync che…
JJ Kavelaars authored
715 return md5.hexdigest()
c281e83 cleaned up some of the error messages to be clearer about what action…
JJ Kavelaars authored
716 if destSize != srcSize:
e6495ea @ijiraq send a better error message
ijiraq authored
717 logging.error("sizes don't match ( %s -> %s ) " % ( src, dest))
c281e83 cleaned up some of the error messages to be clearer about what action…
JJ Kavelaars authored
718 raise IOError(errno.EIO,"sizes don't match",src)
719 return destSize
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
720
721
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
722 def fixURI(self,uri):
723 """given a uri check if the server part is there and if it isn't update that"""
2bc921e Some bug fixxes and moved back to optparse to stay P2.6 compatible
JJ Kavelaars authored
724 from errno import EINVAL
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
725 #logging.debug("trying to fix URL: %s" % ( uri))
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
726 if uri[0:4] != "vos:":
727 uri=self.rootNode+uri
728 parts=urlparse(uri)
729 if parts.scheme!="vos":
2bc921e Some bug fixxes and moved back to optparse to stay P2.6 compatible
JJ Kavelaars authored
730 raise IOError(errno.EINVAL,"Invalid vospace URI",uri)
731 import re
732 ## Check that path name compiles with the standard
733 filename=os.path.basename(parts.path)
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
734 #logging.debug("Checking file name: %s" %( filename))
735 #logging.debug("Result: %s" % (re.match("^[\_\-\(\)\=\+\!\,\;\:\@\&\*\$\.\w\~]*$",filename)))
2b5511d @ijiraq Allow ~ in filenames
ijiraq authored
736 if not re.match("^[\_\-\(\)\=\+\!\,\;\:\@\&\*\$\.\w\~]*$",filename):
2bc921e Some bug fixxes and moved back to optparse to stay P2.6 compatible
JJ Kavelaars authored
737 raise IOError(errno.EINVAL,"Illegal vospace container name",filename)
738
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
739 ## insert the default VOSpace server if none given
740 host=parts.netloc
741 if not host or host=='':
742 host=self.VOSpaceServer
743 path=os.path.normpath(parts.path).strip('/')
744 return "%s://%s/%s" % (parts.scheme, host, path)
745
746
d238aab Cleaned up the error messages and removed some un-needed call of getNode
JJ Kavelaars authored
747 def getNode(self,uri,limit=0):
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
748 """connect to VOSpace and download the definition of vospace node
749
030c36b @ijiraq merge with SVN tree
ijiraq authored
750 uri --- a voSpace node in the format vos:/vospaceName/nodeName
751 limit --- load children nodes in batches of limit
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
752 """
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
753 #logging.debug("Getting node %s" % ( uri))
030c36b @ijiraq merge with SVN tree
ijiraq authored
754 xmlObj=self.open(uri,os.O_RDONLY, limit=0)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
755 dom=ET.parse(xmlObj)
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
756 #logging.debug("%s" %( str(dom)))
030c36b @ijiraq merge with SVN tree
ijiraq authored
757 node = Node(dom.getroot())
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
758 # If this is a container node and the nodlist has not already been set, try to load the children.
759 # this would be better deferred until the children are actually needed, however that would require
760 # access to a connection when the children are accessed, and thats not easy.
7e047f0 @ijiraq added some code comments
ijiraq authored
761 # IF THE CALLER KNOWS THEY DON'T NEED THE CHILDREN THEY CAN SET LIMIT=0 IN THE CALL
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
762 if node.isdir() and limit>0:
87cbbf4 @ijiraq Fixxed vcp error related to directory listsing.
ijiraq authored
763 node._nodeList=[]
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
764 #logging.debug("Loading children")
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
765 nextURI = None
766 again = True
767 while again:
768 again = False
769 getChildrenXMLDoc=self.open(uri,os.O_RDONLY, nextURI=nextURI)
770 getChildrenDOM = ET.parse(getChildrenXMLDoc)
771 for nodesNode in getChildrenDOM.findall(Node.NODES):
772 for child in nodesNode.findall(Node.NODE):
773 if child.get('uri') != nextURI:
774 childNode = node.addChild(child)
775 nextURI = childNode.uri
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
776 #logging.debug("added child %s" % childNode.uri)
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
777 again = True
778 return(node)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
779
780
030c36b @ijiraq merge with SVN tree
ijiraq authored
781 def getNodeURL(self,uri,protocol="https", method='GET', view=None, limit=0, nextURI=None):
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
782 """Split apart the node string into parts and return the correct URL for this node"""
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
783 import urllib
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
784
785 uri = self.fixURI(uri)
786 parts = urlparse(uri)
787 path = parts.path.strip('/')
788 server= Client.VOServers.get(parts.netloc)
789
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
790 #logging.debug("Node URI: %s" %( uri))
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
791
792 if method in ('PUT'):
7e047f0 @ijiraq added some code comments
ijiraq authored
793 ## having a limit is not expected for PUT
794 limit=None
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
795 #logging.debug("PUT structure hardcoded for CADC vospace" )
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
796 ### This part is hard coded for CADC VOSpace...
ce41d51 @ijiraq Made the client more archive agnostic
ijiraq authored
797 return "%s://%s/data/pub/%s/%s" % (protocol, server,self.archive,parts.path.strip('/'))
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
798
799 ### this is a GET so we might have to stick some data onto the URL...
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
800 fields={}
87cbbf4 @ijiraq Fixxed vcp error related to directory listsing.
ijiraq authored
801 if limit is not None:
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
802 fields['limit']=limit
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
803 if view is not None:
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
804 fields['view'] = view
030c36b @ijiraq merge with SVN tree
ijiraq authored
805 if nextURI is not None:
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
806 fields['uri'] = nextURI
87cbbf4 @ijiraq Fixxed vcp error related to directory listsing.
ijiraq authored
807 data=""
808 if len(fields) >0 :
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
809 data="?"+urllib.urlencode(fields)
810 URL = "%s://%s/vospace/nodes/%s%s" % ( protocol, server, parts.path.strip('/'), data)
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
811 #logging.debug("Node URL %s (%s)" % (URL, method))
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
812 return URL
9ec6a01 @ijiraq added vos to vos rename
ijiraq authored
813
814 def move(self,srcURI,destURI):
815 """Move srcUri to targetUri"""
816 transfer=ET.Element("transfer")
817 transfer.attrib['xmlns']=Node.VOSNS
818 transfer.attrib['xmlns:vos']=Node.VOSNS
819 ET.SubElement(transfer,"target").text=self.fixURI(srcURI)
820 ET.SubElement(transfer,"direction").text=self.fixURI(destURI)
821 ET.SubElement(transfer,"keepBytes").text="false"
822
823 con=self.open(srcURI,URL=Client.VOTransfer,mode=os.O_APPEND)
824 con.write(ET.tostring(transfer))
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
825 con.read()
9ec6a01 @ijiraq added vos to vos rename
ijiraq authored
826 if con.resp.status==200:
e03d39c @igable Make the file consistently use 4 spaces for tabs. The file was mixed.
igable authored
827 return True
9ec6a01 @ijiraq added vos to vos rename
ijiraq authored
828 return False
829
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
830
87cbbf4 @ijiraq Fixxed vcp error related to directory listsing.
ijiraq authored
831 def open(self, uri, mode=os.O_RDONLY, view=None, head=False, URL=None, limit=None, nextURI=None,size=None):
030c36b @ijiraq merge with SVN tree
ijiraq authored
832 """Connect to the uri as a VOFile object"""
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
833
834 ### sometimes this is called with mode from ['w', 'r']
835 if type(mode)==str:
836 mode=os.O_RDONLY
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
837
838 # the URL of the connection depends if we are 'getting', 'putting' or 'posting' data
839 method=None
840 if mode == os.O_RDONLY:
841 method="GET"
842 elif mode & ( os.O_WRONLY | os.O_CREAT) :
843 method="PUT"
844 elif mode & os.O_APPEND :
845 method="POST"
846 elif mode & os.O_TRUNC:
847 method="DELETE"
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
848 if head:
849 method="HEAD"
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
850 if not method:
2bc921e Some bug fixxes and moved back to optparse to stay P2.6 compatible
JJ Kavelaars authored
851 raise IOError(errno.EOPNOTSUPP,"Invalid access mode", mode)
9ec6a01 @ijiraq added vos to vos rename
ijiraq authored
852 if URL is None:
030c36b @ijiraq merge with SVN tree
ijiraq authored
853 URL=self.getNodeURL(uri, method=method, view=view,limit=limit,nextURI=nextURI)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
854 logging.debug(URL)
5ba5b61 Modified vos.py to allow 'size' of file to be sent, used in vsync che…
JJ Kavelaars authored
855 return VOFile(URL,self.conn,method=method,size=size)
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
856
b755f98 Added new function to push only 'changed' or 'deleted' properties to …
JJ Kavelaars authored
857
858 def addProps(self,node):
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
859 """Given a node structure do a POST of the XML to the VOSpace to update the node properties"""
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
860 #logging.debug("Updating %s" % ( node.name))
861 #logging.debug(str(node.props))
b755f98 Added new function to push only 'changed' or 'deleted' properties to …
JJ Kavelaars authored
862 ## Get a copy of what's on the server
863 storedNode=self.getNode(node.uri)
864 for prop in storedNode.props:
865 if prop in node.props and storedNode.props[prop]==node.props[prop] and node.props[prop] is not None:
866 del(node.props[prop])
867 for properties in node.node.findall(Node.PROPERTIES):
868 node.node.remove(properties)
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
869 #logging.debug(str(node))
b755f98 Added new function to push only 'changed' or 'deleted' properties to …
JJ Kavelaars authored
870 properties=ET.Element(Node.PROPERTIES)
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
871 #logging.debug(node.props)
b755f98 Added new function to push only 'changed' or 'deleted' properties to …
JJ Kavelaars authored
872 for prop in node.props:
873 property=ET.SubElement(properties,Node.PROPERTY,attrib={'readOnly': 'false', 'uri': "%s#%s" % (Node.IVOAURL, prop)})
874 if node.props[prop] is None:
875 property.attrib['xsi:nil']='true'
4b6c3be @ijiraq added xmlns:xsi to changeProp
ijiraq authored
876 property.attrib["xmlns:xsi"]=Node.XSINS
b755f98 Added new function to push only 'changed' or 'deleted' properties to …
JJ Kavelaars authored
877 property.text=""
878 else:
879 property.text=node.props[prop]
880 node.node.insert(0,properties)
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
881 #logging.debug(str(node))
b755f98 Added new function to push only 'changed' or 'deleted' properties to …
JJ Kavelaars authored
882 f=self.open(node.uri,mode=os.O_APPEND)
883 f.write(str(node))
884 f.close()
885 return
886
887 def update(self,node):
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
888 ##logging.debug(str(node))
b755f98 Added new function to push only 'changed' or 'deleted' properties to …
JJ Kavelaars authored
889 ## Now, remove from the node.node properties those entries that
890 ## match the stored values
891 if 1==2:
892 for properties in storedNode.node.findall(Node.PROPERTIES):
893 for prop in properties.findall(Node.PROPERTY):
894 propName = prop.get('uri',None)
895 if propName is not None:
896 for properties2 in node.node.findall(Node.PROPERTIES):
897 for prop2 in properties2.findall(Node.PROPERTY):
898 propName2=prop2.get('uri',None)
899 if propName == propName2:
900 if prop2.text==prop.text:
901 properties2.remove(prop2)
902 #logging.debug(str(node))
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
903 f=self.open(node.uri,mode=os.O_APPEND)
904 f.write(str(node))
905 f.close()
906
907 def mkdir(self,uri):
908 node = Node(self.fixURI(uri),nodeType="vos:ContainerNode")
909 URL=self.getNodeURL(uri)
910 f=VOFile(URL,self.conn,method="PUT")
911 f.write(str(node))
af3e34a Moved the loop until connection succeeds into the connection object c…
JJ Kavelaars authored
912 return f.close()
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
913
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
914 def delete(self,uri):
915 """Delete the node"""
916 logging.debug("%s" % (uri))
af3e34a Moved the loop until connection succeeds into the connection object c…
JJ Kavelaars authored
917 return self.open(uri,mode=os.O_TRUNC).close()
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
918
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
919 def listdir(self,uri):
920 """Walk through the directory structure a al os.walk"""
44861f9 commented out a subtantial fraction of the debug logging
JJ Kavelaars authored
921 #logging.debug("getting a listing of %s " % ( uri))
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
922 names=[]
87cbbf4 @ijiraq Fixxed vcp error related to directory listsing.
ijiraq authored
923 for node in self.getNode(uri,limit=1).getNodeList():
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
924 names.append(node.name)
925 return names
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
926
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
927 def isdir(self,uri):
928 """Check to see if this given uri points at a containerNode."""
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
929 try:
930 return self.getNode(uri,limit=0).isdir()
931 except:
932 return False
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
933
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
934 def isfile(self,uri):
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
935 try:
936 return self.status(uri)
937 except:
938 return False
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
939
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
940 def access(self,uri,mode=os.O_RDONLY):
941 """Test for existance"""
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
942 try:
e504dd0 @ijiraq made access use getNode instead of openning the data view
ijiraq authored
943 dum=self.getNode(uri)
944 return True
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
945 except Exception as e:
e504dd0 @ijiraq made access use getNode instead of openning the data view
ijiraq authored
946 return False
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
947
47de47e @ijiraq do a retry on status 503
ijiraq authored
948 def status(self,uri,code=[200,303,302]):
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
949 """Check to see if this given uri points at a containerNode.
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
950
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
951 This is done by checking the view=data header and seeing if you get an error.
952 """
35bd810 New VOSpace release removed MD5 and length from 0-length files. Modi…
JJ Kavelaars authored
953 return self.open(uri,view='data',head=True).close(code=code)
f7eefef @ijiraq fixxed up the copy interaction
ijiraq authored
954
6d35121 Intial project commit after export from googlecode
JJ Kavelaars authored
955
Something went wrong with that request. Please try again.