Support for checking status of CF Invalidation Requests [cfinvalinfo].

* s3cmd, S3/CloudFront.py, S3/S3Uri.py: Support for checking
  status of CF Invalidation Requests [cfinvalinfo].

git-svn-id: https://s3tools.svn.sourceforge.net/svnroot/s3tools/s3cmd/trunk@474 830e0280-6d2a-0410-9c65-932aecc39d9d

ChangeLog

@@ -1,5 +1,7 @@
 2011-04-10  Michal Ludvig  <mludvig@logix.net.nz>
 
+	* s3cmd, S3/CloudFront.py, S3/S3Uri.py: Support for checking 
+	  status of CF Invalidation Requests [cfinvalinfo].
 	* s3cmd, S3/CloudFront.py, S3/Config.py: Support for CloudFront
 	  invalidation using [sync --cf-invalidate] command.
 	* S3/Utils.py: getDictFromTree() now recurses into

S3/CloudFront.py

@@ -191,6 +191,63 @@ def __str__(self):
 			tree.append(logging_el)
 		return ET.tostring(tree)
 
+class Invalidation(object):
+	## Example:
+	##
+	## <Invalidation xmlns="http://cloudfront.amazonaws.com/doc/2010-11-01/">
+	##   <Id>id</Id>
+	##   <Status>status</Status>
+	##   <CreateTime>date</CreateTime>
+	##   <InvalidationBatch>
+	##       <Path>/image1.jpg</Path>
+	##       <Path>/image2.jpg</Path>
+	##       <Path>/videos/movie.flv</Path>
+	##       <CallerReference>my-batch</CallerReference>
+	##   </InvalidationBatch>
+	## </Invalidation>
+
+	def __init__(self, xml):
+		tree = getTreeFromXml(xml)
+		if tree.tag != "Invalidation":
+			raise ValueError("Expected <Invalidation /> xml, got: <%s />" % tree.tag)
+		self.parse(tree)
+
+	def parse(self, tree):
+		self.info = getDictFromTree(tree)
+
+	def __str__(self):
+		return str(self.info)
+
+class InvalidationList(object):
+	## Example:
+	##
+	## <InvalidationList>
+	##   <Marker/>
+	##   <NextMarker>Invalidation ID</NextMarker>
+	##   <MaxItems>2</MaxItems>
+	##   <IsTruncated>true</IsTruncated>
+	##   <InvalidationSummary>
+	##     <Id>[Second Invalidation ID]</Id>
+	##     <Status>Completed</Status>
+	##   </InvalidationSummary>
+	##   <InvalidationSummary>
+	##     <Id>[First Invalidation ID]</Id>
+	##     <Status>Completed</Status>
+	##   </InvalidationSummary>
+	## </InvalidationList>
+
+	def __init__(self, xml):
+		tree = getTreeFromXml(xml)
+		if tree.tag != "InvalidationList":
+			raise ValueError("Expected <InvalidationList /> xml, got: <%s />" % tree.tag)
+		self.parse(tree)
+
+	def parse(self, tree):
+		self.info = getDictFromTree(tree)
+
+	def __str__(self):
+		return str(self.info)
+
 class InvalidationBatch(object):
 	## Example:
 	##
@@ -240,7 +297,7 @@ class CloudFront(object):
 		"SetDistConfig" : { 'method' : "PUT", 'resource' : "/%(dist_id)s/config" },
 		"Invalidate" : { 'method' : "POST", 'resource' : "/%(dist_id)s/invalidation" },
 		"GetInvalList" : { 'method' : "GET", 'resource' : "/%(dist_id)s/invalidation" },
-		"GetInvalStatus" : { 'method' : "GET", 'resource' : "/%(dist_id)s/invalidation/%(invalidation_id)s" },
+		"GetInvalInfo" : { 'method' : "GET", 'resource' : "/%(dist_id)s/invalidation/%(request_id)s" },
 	}
 
 	## Maximum attempts of re-issuing failed requests
@@ -384,18 +441,38 @@ def InvalidateObjects(self, uri, paths):
 		debug("InvalidateObjects(): request_body: %s" % invalbatch)
 		response = self.send_request("Invalidate", dist_id = cfuri.dist_id(),
 		                             body = str(invalbatch))
+		response['dist_id'] = cfuri.dist_id()
+		if response['status'] == 201:
+			inval_info = Invalidation(response['data']).info
+			response['request_id'] = inval_info['Id']
 		debug("InvalidateObjects(): response: %s" % response)
-		return response, invalbatch.get_reference()
+		return response
+
+	def GetInvalList(self, cfuri):
+		if cfuri.type != "cf":
+			raise ValueError("Expected CFUri instead of: %s" % cfuri)
+		response = self.send_request("GetInvalList", dist_id = cfuri.dist_id())
+		response['inval_list'] = InvalidationList(response['data'])
+		return response
+
+	def GetInvalInfo(self, cfuri):
+		if cfuri.type != "cf":
+			raise ValueError("Expected CFUri instead of: %s" % cfuri)
+		if cfuri.request_id() is None:
+			raise ValueError("Expected CFUri with Request ID")
+		response = self.send_request("GetInvalInfo", dist_id = cfuri.dist_id(), request_id = cfuri.request_id())
+		response['inval_status'] = Invalidation(response['data'])
+		return response
 
 	## --------------------------------------------------
 	## Low-level methods for handling CloudFront requests
 	## --------------------------------------------------
 
-	def send_request(self, op_name, dist_id = None, body = None, headers = {}, retries = _max_retries):
+	def send_request(self, op_name, dist_id = None, request_id = None, body = None, headers = {}, retries = _max_retries):
 		operation = self.operations[op_name]
 		if body:
 			headers['content-type'] = 'text/plain'
-		request = self.create_request(operation, dist_id, headers)
+		request = self.create_request(operation, dist_id, request_id, headers)
 		conn = self.get_connection()
 		debug("send_request(): %s %s" % (request['method'], request['resource']))
 		conn.request(request['method'], request['resource'], body, request['headers'])
@@ -425,9 +502,9 @@ def send_request(self, op_name, dist_id = None, body = None, headers = {}, retri
 
 		return response
 
-	def create_request(self, operation, dist_id = None, headers = None):
+	def create_request(self, operation, dist_id = None, request_id = None, headers = None):
 		resource = cloudfront_resource + (
-		           operation['resource'] % { 'dist_id' : dist_id })
+		           operation['resource'] % { 'dist_id' : dist_id, 'request_id' : request_id })
 
 		if not headers:
 			headers = {}
@@ -635,5 +712,24 @@ def modify(args):
 		pretty_output("Etag", response['headers']['etag'])
 
 	@staticmethod
-	def invalidate(args):
+	def invalinfo(args):
 		cf = CloudFront(Config())
+		cfuris = Cmd._parse_args(args)
+		requests = []
+		for cfuri in cfuris:
+			if cfuri.request_id():
+				requests.append(str(cfuri))
+			else:
+				inval_list = cf.GetInvalList(cfuri)
+				for i in inval_list['inval_list'].info['InvalidationSummary']:
+					requests.append("/".join(["cf:/", cfuri.dist_id(), i["Id"]]))
+		for req in requests:
+			cfuri = S3Uri(req)
+			inval_info = cf.GetInvalInfo(cfuri)
+			st = inval_info['inval_status'].info
+			pretty_output("URI", str(cfuri))
+			pretty_output("Status", st['Status'])
+			pretty_output("Created", st['CreateTime'])
+			pretty_output("Nr of paths", len(st['InvalidationBatch']['Path']))
+			pretty_output("Reference", st['InvalidationBatch']['CallerReference'])
+			output("")

S3/S3Uri.py

@@ -158,19 +158,26 @@ def dirname(self):
 
 class S3UriCloudFront(S3Uri):
 	type = "cf"
-	_re = re.compile("^cf://([^/]*)/?", re.IGNORECASE)
+	_re = re.compile("^cf://([^/]*)(/.*)?", re.IGNORECASE)
 	def __init__(self, string):
 		match = self._re.match(string)
 		if not match:
 			raise ValueError("%s: not a CloudFront URI" % string)
 		groups = match.groups()
 		self._dist_id = groups[0]
+		self._request_id = groups[1] != "/" and groups[1] or None
 
 	def dist_id(self):
 		return self._dist_id
 
+	def request_id(self):
+		return self._request_id
+
 	def uri(self):
-		return "/".join(["cf:/", self.dist_id()])
+		uri = "cf://" + self.dist_id()
+		if self.request_id():
+			uri += "/" + self.request_id()
+		return uri
 
 if __name__ == "__main__":
 	uri = S3Uri("s3://bucket/object")

s3cmd

@@ -893,9 +893,10 @@ def cmd_sync_local2remote(args):
 		else:
 			# 'uri' from the last iteration is still valid at this point
 			cf = CloudFront(cfg)
-			result, inval_id = cf.InvalidateObjects(uri, uploaded_objects_list)
-			print result
-			output("Created invalidation request: %s" % inval_id)
+			result = cf.InvalidateObjects(uri, uploaded_objects_list)
+			if result['status'] == 201:
+				output("Created invalidation request for %d paths" % len(uploaded_objects_list)) 
+				output("Check progress with: s3cmd cfinvalinfo cf://%s/%s" % (result['dist_id'], result['request_id']))
 
 def cmd_sync(args):
 	if (len(args) < 2):
@@ -1328,6 +1329,8 @@ def get_commands_list():
 	{"cmd":"cfcreate", "label":"Create CloudFront distribution point", "param":"s3://BUCKET", "func":CfCmd.create, "argc":1},
 	{"cmd":"cfdelete", "label":"Delete CloudFront distribution point", "param":"cf://DIST_ID", "func":CfCmd.delete, "argc":1},
 	{"cmd":"cfmodify", "label":"Change CloudFront distribution point parameters", "param":"cf://DIST_ID", "func":CfCmd.modify, "argc":1},
+	#{"cmd":"cfinval", "label":"Invalidate CloudFront objects", "param":"s3://BUCKET/OBJECT [s3://BUCKET/OBJECT ...]", "func":CfCmd.invalidate, "argc":1},
+	{"cmd":"cfinvalinfo", "label":"Display CloudFront invalidation request(s) status", "param":"cf://DIST_ID[/INVAL_ID]", "func":CfCmd.invalinfo, "argc":1},
 	]
 
 def format_commands(progname, commands_list):