* S3/CloudFront.py: New module for CloudFront support.

* s3cmd, S3/Config.py, S3/Exceptions.py: Wire in CF support.



git-svn-id: https://s3tools.svn.sourceforge.net/svnroot/s3tools/s3cmd/branches/s3cmd-airlock@339 830e0280-6d2a-0410-9c65-932aecc39d9d

ChangeLog

@@ -1,3 +1,8 @@
+2009-01-16  Michal Ludvig  <michal@logix.cz>
+
+	* S3/CloudFront.py: New module for CloudFront support.
+	* s3cmd, S3/Config.py, S3/Exceptions.py: Wire in CF support.
+
 2009-01-13  Michal Ludvig  <michal@logix.cz>
 
 	* TODO: Updated.

S3/CloudFront.py

@@ -0,0 +1,136 @@
+## Amazon CloudFront support
+## Author: Michal Ludvig <michal@logix.cz>
+##         http://www.logix.cz/michal
+## License: GPL Version 2
+
+import base64
+import time
+import httplib
+from logging import debug, info, warning, error
+
+try:
+	from hashlib import md5, sha1
+except ImportError:
+	from md5 import md5
+	import sha as sha1
+import hmac
+
+from Config import Config
+from Exceptions import *
+
+try:
+	import xml.etree.ElementTree as ET
+except ImportError:
+	import elementtree.ElementTree as ET
+
+class Distribution(object):
+	pass
+
+class CloudFront(object):
+	operations = {
+		"Create" : { 'method' : "PUT", 'resource' : "" },
+		"Delete" : { 'method' : "DELETE", 'resource' : "/%(dist_id)s" },
+		"GetList" : { 'method' : "GET", 'resource' : "" },
+		"GetDistInfo" : { 'method' : "GET", 'resource' : "/%(dist_id)s" },
+		"GetDistConfig" : { 'method' : "GET", 'resource' : "/%(dist_id)s/config" },
+		"SetDistConfig" : { 'method' : "PUT", 'resource' : "/%(dist_id)s/config" },
+	}
+
+	## Maximum attempts of re-issuing failed requests
+	_max_retries = 5
+
+	def __init__(self, config):
+		self.config = config
+
+	## --------------------------------------------------
+	## Methods implementing CloudFront API
+	## --------------------------------------------------
+
+	def GetList(self):
+		response = self.send_request("GetList")
+		return response
+
+	## --------------------------------------------------
+	## Low-level methods for handling CloudFront requests
+	## --------------------------------------------------
+
+	def send_request(self, op_name, dist_id = None, body = None, retries = _max_retries):
+		operation = self.operations[op_name]
+		request = self.create_request(operation, dist_id)
+		conn = self.get_connection()
+		conn.request(request['method'], request['resource'], body, request['headers'])
+		http_response = conn.getresponse()
+		response = {}
+		response["status"] = http_response.status
+		response["reason"] = http_response.reason
+		response["headers"] = dict(http_response.getheaders())
+		response["data"] =  http_response.read()
+		conn.close()
+
+		debug("CloudFront: response: %r" % response)
+
+		if response["status"] >= 400:
+			e = CloudFrontError(response)
+			if retries:
+				warning(u"Retrying failed request: %s" % op_name)
+				warning(unicode(e))
+				warning("Waiting %d sec..." % self._fail_wait(retries))
+				time.sleep(self._fail_wait(retries))
+				return self.send_request(op_name, dist_id, body, retries - 1)
+			else:
+				raise e
+
+		if response["status"] < 200 or response["status"] > 299:
+			raise CloudFrontError(response)
+
+		return response
+
+	def create_request(self, operation, dist_id = None, headers = None):
+		resource = self.config.cloudfront_resource + (
+		           operation['resource'] % { 'dist_id' : dist_id })
+
+		if not headers:
+			headers = {}
+
+		if headers.has_key("date"):
+			if not headers.has_key("x-amz-date"):
+				headers["x-amz-date"] = headers["date"]
+			del(headers["date"])
+
+		if not headers.has_key("x-amz-date"):
+			headers["x-amz-date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
+
+		signature = self.sign_request(headers)
+		headers["Authorization"] = "AWS "+self.config.access_key+":"+signature
+
+		request = {}
+		request['resource'] = resource
+		request['headers'] = headers
+		request['method'] = operation['method']
+
+		return request
+
+	def sign_request(self, headers):
+		string_to_sign = headers['x-amz-date']
+		signature = base64.encodestring(hmac.new(self.config.secret_key, string_to_sign, sha1).digest()).strip()
+		debug(u"CloudFront.sign_request('%s') = %s" % (string_to_sign, signature))
+		return signature
+
+	def get_connection(self):
+		if self.config.proxy_host != "":
+			raise ParameterError("CloudFront commands don't work from behind a HTTP proxy")
+		return httplib.HTTPSConnection(self.config.cloudfront_host)
+
+	def _fail_wait(self, retries):
+		# Wait a few seconds. The more it fails the more we wait.
+		return (self._max_retries - retries + 1) * 3
+
+class Cmd(object):
+	"""
+	Class that implements CloudFront commands
+	"""
+
+	@staticmethod
+	def list(args):
+		cf = CloudFront(Config())
+		response = cf.GetList()

S3/Config.py

@@ -17,6 +17,8 @@ class Config(object):
 	host_base = "s3.amazonaws.com"
 	host_bucket = "%(bucket)s.s3.amazonaws.com"
 	simpledb_host = "sdb.amazonaws.com"
+	cloudfront_host = "cloudfront.amazonaws.com"
+	cloudfront_resource = "/2008-06-30/distribution"
 	verbosity = logging.WARNING
 	progress_meter = True
 	progress_class = Progress.ProgressCR

S3/Exceptions.py

@@ -3,7 +3,7 @@
 ##         http://www.logix.cz/michal
 ## License: GPL Version 2
 
-from Utils import getRootTagName, unicodise, deunicodise
+from Utils import getTreeFromXml, unicodise, deunicodise
 from logging import debug, info, warning, error
 
 try:
@@ -30,21 +30,26 @@ def __init__(self, response):
 		if response.has_key("headers"):
 			for header in response["headers"]:
 				debug("HttpHeader: %s: %s" % (header, response["headers"][header]))
-		if response.has_key("data") and getRootTagName(response["data"]) == "Error":
-			tree = ET.fromstring(response["data"])
-			for child in tree.getchildren():
+		if response.has_key("data"):
+			tree = getTreeFromXml(response["data"])
+			error_node = tree
+			if not error_node.tag == "Error":
+				error_node = tree.find(".//Error")
+			for child in error_node.getchildren():
 				if child.text != "":
 					debug("ErrorXML: " + child.tag + ": " + repr(child.text))
 					self.info[child.tag] = child.text
 
 	def __unicode__(self):
-		retval = "%d (%s)" % (self.status, self.reason)
-		try:
-			retval += (": %s" % self.info["Code"])
-		except (AttributeError, KeyError):
-			pass
+		retval = u"%d " % (self.status)
+		retval += (u"(%s)" % (self.info.has_key("Code") and self.info["Code"] or self.reason))
+		if self.info.has_key("Message"):
+			retval += (u": %s" % self.info["Message"])
 		return retval
 
+class CloudFrontError(S3Error):
+	pass
+
 class S3UploadError(S3Exception):
 	pass
 

s3cmd

@@ -21,6 +21,9 @@ from optparse import OptionParser, Option, OptionValueError, IndentedHelpFormatt
 from logging import debug, info, warning, error
 from distutils.spawn import find_executable
 
+commands = {}
+commands_list = []
+
 def output(message):
 	sys.stdout.write(message + "\n")
 
@@ -1098,8 +1101,8 @@ def process_exclude_from_file(exf, exclude_array):
 		debug(u"adding rule: %s" % ex)
 		exclude_array.append(ex)
 
-commands = {}
-commands_list = [
+def get_commands_list():
+	return [
 	{"cmd":"mb", "label":"Make bucket", "param":"s3://BUCKET", "func":cmd_bucket_create, "argc":1},
 	{"cmd":"rb", "label":"Remove bucket", "param":"s3://BUCKET", "func":cmd_bucket_delete, "argc":1},
 	{"cmd":"ls", "label":"List objects or buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_ls, "argc":0},
@@ -1114,6 +1117,12 @@ commands_list = [
 	{"cmd":"cp", "label":"Copy object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_cp, "argc":2},
 	{"cmd":"mv", "label":"Move object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_mv, "argc":2},
 	{"cmd":"setacl", "label":"Modify Access control list for Bucket or Object", "param":"s3://BUCKET[/OBJECT]", "func":cmd_setacl, "argc":1},
+	## CloudFront commands
+	{"cmd":"cflist", "label":"List CloudFront distribution points", "param":"", "func":CfCmd.list, "argc":0},
+	#{"cmd":"cfcreate", "label":"Create CloudFront distribution point", "param":"s3://BUCKET", "func":cmd_cf_create, "argc":1},
+	#{"cmd":"cfdelete", "label":"Delete CloudFront distribution point", "param":"cf://DIST_ID", "func":cmd_cf_delete, "argc":1},
+	#{"cmd":"cfinfo", "label":"Display CloudFront distribution point parameters", "param":"cf://DIST_ID", "func":cmd_cf_info, "argc":1},
+	#{"cmd":"cfmodify", "label":"Change CloudFront distribution point parameters", "param":"cf://DIST_ID", "func":cmd_cf_modify, "argc":1},
 	]
 
 def format_commands(progname):
@@ -1145,6 +1154,9 @@ def main():
 		sys.stderr.write("ERROR: Python 2.4 or higher required, sorry.\n")
 		sys.exit(1)
 
+	global commands_list, commands
+	commands_list = get_commands_list()
+	commands = {}
 	## Populate "commands" from "commands_list"
 	for cmd in commands_list:
 		if cmd.has_key("cmd"):
@@ -1347,8 +1359,6 @@ def main():
 		cmd_func(args)
 	except S3Error, e:
 		error(u"S3 error: %s" % e)
-		if e.info.has_key("Message"):
-			error(e.info['Message'])
 		sys.exit(1)
 	except ParameterError, e:
 		error(u"Parameter problem: %s" % e)
@@ -1367,6 +1377,7 @@ if __name__ == '__main__':
 		from S3.Exceptions import *
 		from S3.Utils import unicodise
 		from S3.Progress import Progress
+		from S3.CloudFront import Cmd as CfCmd
 
 		main()
 		sys.exit(0)