Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

* s3cmd, S3/CloudFront.py, S3/Config.py: Support access

  logging for CloudFront distributions.
* S3/S3.py, S3/Utils.py: Moved some functions to Utils.py
  to make them available to CloudFront.py
* NEWS: Document the above.



git-svn-id: https://s3tools.svn.sourceforge.net/svnroot/s3tools/s3cmd/trunk@415 830e0280-6d2a-0410-9c65-932aecc39d9d
  • Loading branch information...
commit b020ea026a1ec85653a7cef6e73d6e15504978b2 1 parent 79382cd
@mludvig mludvig authored
View
8 ChangeLog
@@ -1,3 +1,11 @@
+2010-06-12 Michal Ludvig <mludvig@logix.net.nz>
+
+ * s3cmd, S3/CloudFront.py, S3/Config.py: Support access
+ logging for CloudFront distributions.
+ * S3/S3.py, S3/Utils.py: Moved some functions to Utils.py
+ to make them available to CloudFront.py
+ * NEWS: Document the above.
+
2010-05-27 Michal Ludvig <mludvig@logix.net.nz>
* S3/S3.py: Fix bucket listing for buckets with
View
2  NEWS
@@ -1,6 +1,8 @@
s3cmd 0.9.9.92 - ???
==============
* Added [accesslog] command. (needs manpage!)
+* Added access logging for CloudFront distributions
+ using [cfmodify --log]
* Added --acl-grant and --acl-revoke (by Timothee Linden).
s3cmd 0.9.9.91 - 2009-10-08
View
44 S3/CloudFront.py
@@ -15,7 +15,7 @@
from Config import Config
from Exceptions import *
-from Utils import getTreeFromXml, appendXmlTextNode, getDictFromTree, dateS3toPython, sign_string
+from Utils import getTreeFromXml, appendXmlTextNode, getDictFromTree, dateS3toPython, sign_string, getBucketFromHostname, getHostnameFromBucket
from S3Uri import S3Uri, S3UriS3
def output(message):
@@ -53,7 +53,7 @@ def uri(self):
class DistributionList(object):
## Example:
##
- ## <DistributionList xmlns="http://cloudfront.amazonaws.com/doc/2008-06-30/">
+ ## <DistributionList xmlns="http://cloudfront.amazonaws.com/doc/2010-06-01/">
## <Marker />
## <MaxItems>100</MaxItems>
## <IsTruncated>false</IsTruncated>
@@ -80,7 +80,7 @@ def parse(self, tree):
class Distribution(object):
## Example:
##
- ## <Distribution xmlns="http://cloudfront.amazonaws.com/doc/2008-06-30/">
+ ## <Distribution xmlns="http://cloudfront.amazonaws.com/doc/2010-06-01/">
## <Id>1234567890ABC</Id>
## <Status>InProgress</Status>
## <LastModifiedTime>2009-01-16T13:07:11.319Z</LastModifiedTime>
@@ -114,10 +114,14 @@ class DistributionConfig(object):
## <CallerReference>s3://somebucket/</CallerReference>
## <Comment>http://somebucket.s3.amazonaws.com/</Comment>
## <Enabled>true</Enabled>
+ ## <Logging>
+ ## <Bucket>bu.ck.et</Bucket>
+ ## <Prefix>/cf-somebucket/</Prefix>
+ ## </Logging>
## </DistributionConfig>
EMPTY_CONFIG = "<DistributionConfig><Origin/><CallerReference/><Enabled>true</Enabled></DistributionConfig>"
- xmlns = "http://cloudfront.amazonaws.com/doc/2008-06-30/"
+ xmlns = "http://cloudfront.amazonaws.com/doc/2010-06-01/"
def __init__(self, xml = None, tree = None):
if not xml:
xml = DistributionConfig.EMPTY_CONFIG
@@ -139,6 +143,16 @@ def parse(self, tree):
self.info['CNAME'] = [cname.lower() for cname in self.info['CNAME']]
if not self.info.has_key("Comment"):
self.info['Comment'] = ""
+ ## Figure out logging - complex node not parsed by getDictFromTree()
+ logging_nodes = tree.findall(".//Logging")
+ if logging_nodes:
+ logging_dict = getDictFromTree(logging_nodes[0])
+ logging_dict['Bucket'], success = getBucketFromHostname(logging_dict['Bucket'])
+ if not success:
+ warning("Logging to unparsable bucket name: %s" % logging_dict['Bucket'])
+ self.info['Logging'] = S3UriS3("s3://%(Bucket)s/%(Prefix)s" % logging_dict)
+ else:
+ self.info['Logging'] = None
def __str__(self):
tree = ET.Element("DistributionConfig")
@@ -152,7 +166,11 @@ def __str__(self):
if self.info['Comment']:
appendXmlTextNode("Comment", self.info['Comment'], tree)
appendXmlTextNode("Enabled", str(self.info['Enabled']).lower(), tree)
-
+ if self.info['Logging']:
+ logging_el = ET.Element("Logging")
+ appendXmlTextNode("Bucket", getHostnameFromBucket(self.info['Logging'].bucket()), logging_el)
+ appendXmlTextNode("Prefix", self.info['Logging'].object(), logging_el)
+ tree.append(logging_el)
return ET.tostring(tree)
class CloudFront(object):
@@ -183,7 +201,7 @@ def GetList(self):
## TODO: handle Truncated
return response
- def CreateDistribution(self, uri, cnames_add = [], comment = None):
+ def CreateDistribution(self, uri, cnames_add = [], comment = None, logging = None):
dist_config = DistributionConfig()
dist_config.info['Enabled'] = True
dist_config.info['Origin'] = uri.host_name()
@@ -195,6 +213,8 @@ def CreateDistribution(self, uri, cnames_add = [], comment = None):
for cname in cnames_add:
if dist_config.info['CNAME'].count(cname) == 0:
dist_config.info['CNAME'].append(cname)
+ if logging != None:
+ dist_config.info['Logging'] = S3UriS3(logging)
request_body = str(dist_config)
debug("CreateDistribution(): request_body: %s" % request_body)
response = self.send_request("CreateDist", body = request_body)
@@ -202,7 +222,7 @@ def CreateDistribution(self, uri, cnames_add = [], comment = None):
return response
def ModifyDistribution(self, cfuri, cnames_add = [], cnames_remove = [],
- comment = None, enabled = None):
+ comment = None, enabled = None, logging = None):
if cfuri.type != "cf":
raise ValueError("Expected CFUri instead of: %s" % cfuri)
# Get current dist status (enabled/disabled) and Etag
@@ -219,6 +239,8 @@ def ModifyDistribution(self, cfuri, cnames_add = [], cnames_remove = [],
for cname in cnames_remove:
while dc.info['CNAME'].count(cname) > 0:
dc.info['CNAME'].remove(cname)
+ if logging != None:
+ dc.info['Logging'] = S3UriS3(logging)
response = self.SetDistConfig(cfuri, dc, response['headers']['etag'])
return response
@@ -364,6 +386,7 @@ class Options(object):
cf_cnames_remove = []
cf_comment = None
cf_enable = None
+ cf_logging = None
def option_list(self):
return [opt for opt in dir(self) if opt.startswith("cf_")]
@@ -402,6 +425,7 @@ def info(args):
pretty_output("CNAMEs", ", ".join(dc.info['CNAME']))
pretty_output("Comment", dc.info['Comment'])
pretty_output("Enabled", dc.info['Enabled'])
+ pretty_output("Logging", dc.info['Logging'] or "Disabled")
pretty_output("Etag", response['headers']['etag'])
@staticmethod
@@ -422,7 +446,8 @@ def create(args):
for uri in buckets:
info("Creating distribution from: %s" % uri)
response = cf.CreateDistribution(uri, cnames_add = Cmd.options.cf_cnames_add,
- comment = Cmd.options.cf_comment)
+ comment = Cmd.options.cf_comment,
+ logging = Cmd.options.cf_logging)
d = response['distribution']
dc = d.info['DistributionConfig']
output("Distribution created:")
@@ -462,7 +487,8 @@ def modify(args):
cnames_add = Cmd.options.cf_cnames_add,
cnames_remove = Cmd.options.cf_cnames_remove,
comment = Cmd.options.cf_comment,
- enabled = Cmd.options.cf_enable)
+ enabled = Cmd.options.cf_enable,
+ logging = Cmd.options.cf_logging)
if response['status'] >= 400:
error("Distribution %s could not be modified: %s" % (cfuri, response['reason']))
output("Distribution modified: %s" % cfuri)
View
2  S3/Config.py
@@ -19,7 +19,7 @@ class Config(object):
host_bucket = "%(bucket)s.s3.amazonaws.com"
simpledb_host = "sdb.amazonaws.com"
cloudfront_host = "cloudfront.amazonaws.com"
- cloudfront_resource = "/2008-06-30/distribution"
+ cloudfront_resource = "/2010-06-01/distribution"
verbosity = logging.WARNING
progress_meter = True
progress_class = Progress.ProgressCR
View
45 S3/S3.py
@@ -135,11 +135,11 @@ def get_connection(self, bucket):
return httplib.HTTPConnection(self.get_hostname(bucket))
def get_hostname(self, bucket):
- if bucket and self.check_bucket_name_dns_conformity(bucket):
+ if bucket and check_bucket_name_dns_conformity(bucket):
if self.redir_map.has_key(bucket):
host = self.redir_map[bucket]
else:
- host = self.config.host_bucket % { 'bucket' : bucket }
+ host = getHostnameFromBucket(bucket)
else:
host = self.config.host_base
debug('get_hostname(%s): %s' % (bucket, host))
@@ -149,7 +149,7 @@ def set_hostname(self, bucket, redir_hostname):
self.redir_map[bucket] = redir_hostname
def format_uri(self, resource):
- if resource['bucket'] and not self.check_bucket_name_dns_conformity(resource['bucket']):
+ if resource['bucket'] and not check_bucket_name_dns_conformity(resource['bucket']):
uri = "/%s%s" % (resource['bucket'], resource['uri'])
else:
uri = resource['uri']
@@ -224,9 +224,9 @@ def bucket_create(self, bucket, bucket_location = None):
body += bucket_location
body += "</LocationConstraint></CreateBucketConfiguration>"
debug("bucket_location: " + body)
- self.check_bucket_name(bucket, dns_strict = True)
+ check_bucket_name(bucket, dns_strict = True)
else:
- self.check_bucket_name(bucket, dns_strict = False)
+ check_bucket_name(bucket, dns_strict = False)
if self.config.acl_public:
headers["x-amz-acl"] = "public-read"
request = self.create_request("BUCKET_CREATE", bucket = bucket, headers = headers)
@@ -753,39 +753,4 @@ def recv_file(self, request, stream, labels, start_position = 0, retries = _max_
warning("MD5 signatures do not match: computed=%s, received=%s" % (
response["md5"], response["headers"]["etag"]))
return response
-
- @staticmethod
- def check_bucket_name(bucket, dns_strict = True):
- if dns_strict:
- invalid = re.search("([^a-z0-9\.-])", bucket)
- if invalid:
- raise ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: lowercase us-ascii letters (a-z), digits (0-9), dot (.) and hyphen (-)." % (bucket, invalid.groups()[0]))
- else:
- invalid = re.search("([^A-Za-z0-9\._-])", bucket)
- if invalid:
- raise ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: us-ascii letters (a-z, A-Z), digits (0-9), dot (.), hyphen (-) and underscore (_)." % (bucket, invalid.groups()[0]))
-
- if len(bucket) < 3:
- raise ParameterError("Bucket name '%s' is too short (min 3 characters)" % bucket)
- if len(bucket) > 255:
- raise ParameterError("Bucket name '%s' is too long (max 255 characters)" % bucket)
- if dns_strict:
- if len(bucket) > 63:
- raise ParameterError("Bucket name '%s' is too long (max 63 characters)" % bucket)
- if re.search("-\.", bucket):
- raise ParameterError("Bucket name '%s' must not contain sequence '-.' for DNS compatibility" % bucket)
- if re.search("\.\.", bucket):
- raise ParameterError("Bucket name '%s' must not contain sequence '..' for DNS compatibility" % bucket)
- if not re.search("^[0-9a-z]", bucket):
- raise ParameterError("Bucket name '%s' must start with a letter or a digit" % bucket)
- if not re.search("[0-9a-z]$", bucket):
- raise ParameterError("Bucket name '%s' must end with a letter or a digit" % bucket)
- return True
-
- @staticmethod
- def check_bucket_name_dns_conformity(bucket):
- try:
- return S3.check_bucket_name(bucket, dns_strict = True)
- except ParameterError:
- return False
__all__.append("S3")
View
61 S3/Utils.py
@@ -119,7 +119,9 @@ def appendXmlTextNode(tag_name, text, parent):
created Node to 'parent' element if given.
Returns the newly created Node.
"""
- parent.append(xmlTextNode(tag_name, text))
+ el = xmlTextNode(tag_name, text)
+ parent.append(el)
+ return el
__all__.append("appendXmlTextNode")
def dateS3toPython(date):
@@ -317,3 +319,60 @@ def sign_string(string_to_sign):
#debug("signature: %s" % signature)
return signature
__all__.append("sign_string")
+
+def check_bucket_name(bucket, dns_strict = True):
+ if dns_strict:
+ invalid = re.search("([^a-z0-9\.-])", bucket)
+ if invalid:
+ raise ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: lowercase us-ascii letters (a-z), digits (0-9), dot (.) and hyphen (-)." % (bucket, invalid.groups()[0]))
+ else:
+ invalid = re.search("([^A-Za-z0-9\._-])", bucket)
+ if invalid:
+ raise ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: us-ascii letters (a-z, A-Z), digits (0-9), dot (.), hyphen (-) and underscore (_)." % (bucket, invalid.groups()[0]))
+
+ if len(bucket) < 3:
+ raise ParameterError("Bucket name '%s' is too short (min 3 characters)" % bucket)
+ if len(bucket) > 255:
+ raise ParameterError("Bucket name '%s' is too long (max 255 characters)" % bucket)
+ if dns_strict:
+ if len(bucket) > 63:
+ raise ParameterError("Bucket name '%s' is too long (max 63 characters)" % bucket)
+ if re.search("-\.", bucket):
+ raise ParameterError("Bucket name '%s' must not contain sequence '-.' for DNS compatibility" % bucket)
+ if re.search("\.\.", bucket):
+ raise ParameterError("Bucket name '%s' must not contain sequence '..' for DNS compatibility" % bucket)
+ if not re.search("^[0-9a-z]", bucket):
+ raise ParameterError("Bucket name '%s' must start with a letter or a digit" % bucket)
+ if not re.search("[0-9a-z]$", bucket):
+ raise ParameterError("Bucket name '%s' must end with a letter or a digit" % bucket)
+ return True
+__all__.append("check_bucket_name")
+
+def check_bucket_name_dns_conformity(bucket):
+ try:
+ return check_bucket_name(bucket, dns_strict = True)
+ except ParameterError:
+ return False
+__all__.append("check_bucket_name_dns_conformity")
+
+def getBucketFromHostname(hostname):
+ """
+ bucket, success = getBucketFromHostname(hostname)
+
+ Only works for hostnames derived from bucket names
+ using Config.host_bucket pattern.
+
+ Returns bucket name and a boolean success flag.
+ """
+
+ # Create RE pattern from Config.host_bucket
+ pattern = Config.Config().host_bucket % { 'bucket' : '(?P<bucket>.*)' }
+ m = re.match(pattern, hostname)
+ if not m:
+ return (hostname, False)
+ return m.groups()[0], True
+__all__.append("getBucketFromHostname")
+
+def getHostnameFromBucket(bucket):
+ return Config.Config().host_bucket % { 'bucket' : bucket }
+__all__.append("getHostnameFromBucket")
View
5 s3cmd
@@ -1599,7 +1599,7 @@ def main():
optparser.add_option( "--bucket-location", dest="bucket_location", help="Datacentre to create bucket in. As of now the datacenters are: US (default), EU, us-west-1, and ap-southeast-1")
optparser.add_option( "--reduced-redundancy", "--rr", dest="reduced_redundancy", action="store_true", help="Store object with 'Reduced redundancy'. Lower per-GB price. [put, cp, mv]")
- optparser.add_option( "--log-target-prefix", dest="log_target_prefix", help="Target prefix for access logs (S3 URI)")
+ optparser.add_option( "--log-target-prefix", dest="log_target_prefix", help="Target prefix for access logs (S3 URI) (for [cfmodify] and [accesslog] commands)")
optparser.add_option("-m", "--mime-type", dest="default_mime_type", type="mimetype", metavar="MIME/TYPE", help="Default MIME-type to be set for objects stored.")
optparser.add_option("-M", "--guess-mime-type", dest="guess_mime_type", action="store_true", help="Guess MIME-type of files by their extension. Falls back to default MIME-Type as specified by --mime-type option")
@@ -1719,6 +1719,9 @@ def main():
## CloudFront's cf_enable and Config's enable share the same --enable switch
options.cf_enable = options.enable
+ ## CloudFront's cf_logging and Config's log_target_prefix share the same --log-target-prefix switch
+ options.cf_logging = options.log_target_prefix
+
## Update CloudFront options if some were set
for option in CfCmd.options.option_list():
try:
Please sign in to comment.
Something went wrong with that request. Please try again.