Skip to content


Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

410 lines (356 sloc) 12.986 kb
## Amazon S3 manager
## Author: Michal Ludvig <>
## License: GPL Version 2
import datetime
import os
import sys
import time
import re
import string
import random
import rfc822
import hmac
import base64
import errno
from logging import debug, info, warning, error
import Config
import Exceptions
# hashlib backported to python 2.4 / 2.5 is not compatible with hmac!
if sys.version_info[0] == 2 and sys.version_info[1] < 6:
from md5 import md5
import sha as sha1
from hashlib import md5, sha1
import xml.etree.ElementTree as ET
except ImportError:
import elementtree.ElementTree as ET
from xml.parsers.expat import ExpatError
__all__ = []
def parseNodes(nodes):
## WARNING: Ignores text nodes from mixed xml/text.
## For instance <tag1>some text<tag2>other text</tag2></tag1>
## will be ignore "some text" node
retval = []
for node in nodes:
retval_item = {}
for child in node.getchildren():
name = child.tag
if child.getchildren():
retval_item[name] = parseNodes([child])
retval_item[name] = node.findtext(".//%s" % child.tag)
return retval
def stripNameSpace(xml):
removeNameSpace(xml) -- remove top-level AWS namespace
r = re.compile('^(<?[^>]+?>\s?)(<\w+) xmlns=[\'"](http://[^\'"]+)[\'"](.*)', re.MULTILINE)
if r.match(xml):
xmlns = r.match(xml).groups()[2]
xml = r.sub("\\1\\2\\4", xml)
xmlns = None
return xml, xmlns
def getTreeFromXml(xml):
xml, xmlns = stripNameSpace(xml)
tree = ET.fromstring(xml)
if xmlns:
tree.attrib['xmlns'] = xmlns
return tree
except ExpatError, e:
raise Exceptions.ParameterError("Bucket contains invalid filenames. Please run: s3cmd fixbucket s3://your-bucket/")
def getListFromXml(xml, node):
tree = getTreeFromXml(xml)
nodes = tree.findall('.//%s' % (node))
return parseNodes(nodes)
def getDictFromTree(tree):
ret_dict = {}
for child in tree.getchildren():
if child.getchildren():
## Complex-type child. Recurse
content = getDictFromTree(child)
content = child.text
if ret_dict.has_key(child.tag):
if not type(ret_dict[child.tag]) == list:
ret_dict[child.tag] = [ret_dict[child.tag]]
ret_dict[child.tag].append(content or "")
ret_dict[child.tag] = content or ""
return ret_dict
def getTextFromXml(xml, xpath):
tree = getTreeFromXml(xml)
if tree.tag.endswith(xpath):
return tree.text
return tree.findtext(xpath)
def getRootTagName(xml):
tree = getTreeFromXml(xml)
return tree.tag
def xmlTextNode(tag_name, text):
el = ET.Element(tag_name)
el.text = unicode(text)
return el
def appendXmlTextNode(tag_name, text, parent):
Creates a new <tag_name> Node and sets
its content to 'text'. Then appends the
created Node to 'parent' element if given.
Returns the newly created Node.
el = xmlTextNode(tag_name, text)
return el
def dateS3toPython(date):
date = re.compile("(\.\d*)?Z").sub(".000Z", date)
return time.strptime(date, "%Y-%m-%dT%H:%M:%S.000Z")
def dateS3toUnix(date):
## FIXME: This should be timezone-aware.
## Currently the argument to strptime() is GMT but mktime()
## treats it as "localtime". Anyway...
return time.mktime(dateS3toPython(date))
def dateRFC822toPython(date):
return rfc822.parsedate(date)
def dateRFC822toUnix(date):
return time.mktime(dateRFC822toPython(date))
def formatSize(size, human_readable = False, floating_point = False):
size = floating_point and float(size) or int(size)
if human_readable:
coeffs = ['k', 'M', 'G', 'T']
coeff = ""
while size > 2048:
size /= 1024
coeff = coeffs.pop(0)
return (size, coeff)
return (size, "")
def formatDateTime(s3timestamp):
import pytz
timezone = pytz.timezone(os.environ.get('TZ', 'UTC'))
utc_dt = datetime.datetime(*dateS3toPython(s3timestamp)[0:6], tzinfo=pytz.timezone('UTC'))
dt_object = utc_dt.astimezone(timezone)
except ImportError:
dt_object = datetime.datetime(*dateS3toPython(s3timestamp)[0:6])
return dt_object.strftime("%Y-%m-%d %H:%M")
def convertTupleListToDict(list):
retval = {}
for tuple in list:
retval[tuple[0]] = tuple[1]
return retval
_rnd_chars = string.ascii_letters+string.digits
_rnd_chars_len = len(_rnd_chars)
def rndstr(len):
retval = ""
while len > 0:
retval += _rnd_chars[random.randint(0, _rnd_chars_len-1)]
len -= 1
return retval
def mktmpsomething(prefix, randchars, createfunc):
old_umask = os.umask(0077)
tries = 5
while tries > 0:
dirname = prefix + rndstr(randchars)
except OSError, e:
if e.errno != errno.EEXIST:
tries -= 1
return dirname
def mktmpdir(prefix = "/tmp/tmpdir-", randchars = 10):
return mktmpsomething(prefix, randchars, os.mkdir)
def mktmpfile(prefix = "/tmp/tmpfile-", randchars = 20):
createfunc = lambda filename : os.close(, os.O_CREAT | os.O_EXCL))
return mktmpsomething(prefix, randchars, createfunc)
def hash_file(filename):
"""Given filename, return dict with hash types as keys, hashes as values"""
import hashlib
md5 = hashlib.md5()
f = open(filename, "rb")
while True:
# Hash 32kB chunks
data =*1024)
if not data:
return dict(md5=md5.hexdigest(), sha1=sha1.hexdigest(), sha256=sha256.hexdigest())
def hash_file_md5(filename):
h = md5()
f = open(filename, "rb")
while True:
# Hash 32kB chunks
data =*1024)
if not data:
return h.hexdigest()
def mkdir_with_parents(dir_name):
Create directory 'dir_name' with all parent directories
Returns True on success, False otherwise.
pathmembers = dir_name.split(os.sep)
tmp_stack = []
while pathmembers and not os.path.isdir(os.sep.join(pathmembers)):
while tmp_stack:
cur_dir = os.sep.join(pathmembers)
debug("mkdir(%s)" % cur_dir)
except (OSError, IOError), e:
warning("%s: can not make directory: %s" % (cur_dir, e.strerror))
return False
except Exception, e:
warning("%s: %s" % (cur_dir, e))
return False
return True
def unicodise(string, encoding = None, errors = "replace"):
Convert 'string' to Unicode or raise an exception.
if not encoding:
encoding = Config.Config().encoding
if type(string) == unicode:
return string
debug("Unicodising %r using %s" % (string, encoding))
return string.decode(encoding, errors)
except UnicodeDecodeError:
raise UnicodeDecodeError("Conversion to unicode failed: %r" % string)
def deunicodise(string, encoding = None, errors = "replace"):
Convert unicode 'string' to <type str>, by default replacing
all invalid characters with '?' or raise an exception.
if not encoding:
encoding = Config.Config().encoding
if type(string) != unicode:
return str(string)
debug("DeUnicodising %r using %s" % (string, encoding))
return string.encode(encoding, errors)
except UnicodeEncodeError:
raise UnicodeEncodeError("Conversion from unicode failed: %r" % string)
def unicodise_safe(string, encoding = None):
Convert 'string' to Unicode according to current encoding
and replace all invalid characters with '?'
return unicodise(deunicodise(string, encoding), encoding).replace(u'\ufffd', '?')
def replace_nonprintables(string):
Replaces all non-printable characters 'ch' in 'string'
where ord(ch) <= 26 with ^@, ^A, ... ^Z
new_string = ""
modified = 0
for c in string:
o = ord(c)
if (o <= 31):
new_string += "^" + chr(ord('@') + o)
modified += 1
elif (o == 127):
new_string += "^?"
modified += 1
new_string += c
if modified and Config.Config().urlencoding_mode != "fixbucket":
warning("%d non-printable characters replaced in: %s" % (modified, new_string))
return new_string
def sign_string(string_to_sign):
#debug("string_to_sign: %s" % string_to_sign)
signature = base64.encodestring(, string_to_sign, sha1).digest()).strip()
#debug("signature: %s" % signature)
return signature
def check_bucket_name(bucket, dns_strict = True):
if dns_strict:
invalid ="([^a-z0-9\.-])", bucket)
if invalid:
raise Exceptions.ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: lowercase us-ascii letters (a-z), digits (0-9), dot (.) and hyphen (-)." % (bucket, invalid.groups()[0]))
invalid ="([^A-Za-z0-9\._-])", bucket)
if invalid:
raise Exceptions.ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: us-ascii letters (a-z, A-Z), digits (0-9), dot (.), hyphen (-) and underscore (_)." % (bucket, invalid.groups()[0]))
if len(bucket) < 3:
raise Exceptions.ParameterError("Bucket name '%s' is too short (min 3 characters)" % bucket)
if len(bucket) > 255:
raise Exceptions.ParameterError("Bucket name '%s' is too long (max 255 characters)" % bucket)
if dns_strict:
if len(bucket) > 63:
raise Exceptions.ParameterError("Bucket name '%s' is too long (max 63 characters)" % bucket)
if"-\.", bucket):
raise Exceptions.ParameterError("Bucket name '%s' must not contain sequence '-.' for DNS compatibility" % bucket)
if"\.\.", bucket):
raise Exceptions.ParameterError("Bucket name '%s' must not contain sequence '..' for DNS compatibility" % bucket)
if not"^[0-9a-z]", bucket):
raise Exceptions.ParameterError("Bucket name '%s' must start with a letter or a digit" % bucket)
if not"[0-9a-z]$", bucket):
raise Exceptions.ParameterError("Bucket name '%s' must end with a letter or a digit" % bucket)
return True
def check_bucket_name_dns_conformity(bucket):
return check_bucket_name(bucket, dns_strict = True)
except Exceptions.ParameterError:
return False
def getBucketFromHostname(hostname):
bucket, success = getBucketFromHostname(hostname)
Only works for hostnames derived from bucket names
using Config.host_bucket pattern.
Returns bucket name and a boolean success flag.
# Create RE pattern from Config.host_bucket
pattern = Config.Config().host_bucket % { 'bucket' : '(?P<bucket>.*)' }
m = re.match(pattern, hostname)
if not m:
return (hostname, False)
return m.groups()[0], True
def getHostnameFromBucket(bucket):
return Config.Config().host_bucket % { 'bucket' : bucket }
# vim:et:ts=4:sts=4:ai
Jump to Line
Something went wrong with that request. Please try again.