2007-05-27 Michal Ludvig <michal@logix.cz>

	* Support for on-the-fly GPG encryption.



git-svn-id: https://s3tools.svn.sourceforge.net/svnroot/s3tools/s3cmd/trunk@121 830e0280-6d2a-0410-9c65-932aecc39d9d

ChangeLog

@@ -1,3 +1,7 @@
+2007-05-27  Michal Ludvig  <michal@logix.cz>
+
+	* Support for on-the-fly GPG encryption.
+
 2007-05-26  Michal Ludvig  <michal@logix.cz>
 
 	* s3cmd.1: Add info about "s3cmd du" command.

S3/Config.py

@@ -19,6 +19,11 @@ class Config(object):
 	human_readable_sizes = False
 	force = False
 	acl_public = False
+	encrypt = False
+	gpg_passphrase = ""
+	gpg_command = "/usr/bin/gpg"
+	gpg_encrypt = "%(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s"
+	gpg_decrypt = "%(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s"
 
 	## Creating a singleton
 	def __new__(self, configfile = None):
@@ -109,8 +114,8 @@ def parse_file(self, file, sections = []):
 				if r_quotes.match(data["value"]):
 					data["value"] = data["value"][1:-1]
 				self.__setitem__(data["key"], data["value"])
-				if data["key"] in ("access_key", "secret_key"):
-					print_value = (data["value"][:3]+"...%d_chars..."+data["value"][-2:]) % (len(data["value"]) - 4)
+				if data["key"] in ("access_key", "secret_key", "gpg_passphrase"):
+					print_value = (data["value"][:2]+"...%d_chars..."+data["value"][-1:]) % (len(data["value"]) - 3)
 				else:
 					print_value = data["value"]
 				debug("ConfigParser: %s->%s" % (data["key"], print_value))

S3/S3.py

@@ -109,7 +109,7 @@ def bucket_delete(self, bucket):
 		response = self.send_request(request)
 		return response
 
-	def object_put(self, filename, bucket, object):
+	def object_put(self, filename, bucket, object, extra_headers = None):
 		if not os.path.isfile(filename):
 			raise ParameterError("%s is not a regular file" % filename)
 		try:
@@ -118,6 +118,8 @@ def object_put(self, filename, bucket, object):
 		except IOError, e:
 			raise ParameterError("%s: %s" % (filename, e.strerror))
 		headers = SortedDict()
+		if extra_headers:
+			headers.update(extra_headers)
 		headers["content-length"] = size
 		if self.config.acl_public:
 			headers["x-amz-acl"] = "public-read"
@@ -143,10 +145,10 @@ def object_delete(self, bucket, object):
 		response = self.send_request(request)
 		return response
 
-	def object_put_uri(self, filename, uri):
+	def object_put_uri(self, filename, uri, extra_headers = None):
 		if uri.type != "s3":
 			raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
-		return self.object_put(filename, uri.bucket(), uri.object())
+		return self.object_put(filename, uri.bucket(), uri.object(), extra_headers)
 
 	def object_get_uri(self, uri, filename):
 		if uri.type != "s3":

S3/Utils.py

@@ -3,9 +3,12 @@
 ##         http://www.logix.cz/michal
 ## License: GPL Version 2
 
+import os
 import time
 import re
 import elementtree.ElementTree as ET
+import string
+import random
 
 def parseNodes(nodes, xmlns = ""):
 	retval = []
@@ -68,3 +71,36 @@ def convertTupleListToDict(list):
 		retval[tuple[0]] = tuple[1]
 	return retval
 
+
+_rnd_chars = string.ascii_letters+string.digits
+_rnd_chars_len = len(_rnd_chars)
+def rndstr(len):
+	retval = ""
+	while len > 0:
+		retval += _rnd_chars[random.randint(0, _rnd_chars_len-1)]
+		len -= 1
+	return retval
+
+def mktmpsomething(prefix, randchars, createfunc):
+	old_umask = os.umask(0077)
+	tries = 5
+	while tries > 0:
+		dirname = prefix + rndstr(randchars)
+		try:
+			createfunc(dirname)
+			break
+		except OSError, e:
+			if e.errno != errno.EEXIST:
+				os.umask(old_umask)
+				raise
+		tries -= 1
+
+	os.umask(old_umask)
+	return dirname
+
+def mktmpdir(prefix = "/tmp/tmpdir-", randchars = 10):
+	return mktmpsomething(prefix, randchars, os.mkdir)
+
+def mktmpfile(prefix = "/tmp/tmpfile-", randchars = 20):
+	createfunc = lambda filename : os.close(os.open(filename, os.O_CREAT | os.O_EXCL))
+	return mktmpsomething(prefix, randchars, createfunc)

s3cmd

@@ -19,6 +19,7 @@ from S3 import PkgInfo
 from S3.S3 import *
 from S3.Config import Config
 from S3.S3Uri import *
+from S3 import Utils
 
 
 def output(message):
@@ -172,14 +173,21 @@ def cmd_object_put(args):
 		uri_arg_final = str(uri)
 		if len(files) > 1 or uri.object() == "":
 			uri_arg_final += os.path.basename(file)
-
+		
 		uri_final = S3Uri(uri_arg_final)
-		response = s3.object_put_uri(file, uri_final)
+		extra_headers = {}
+		real_filename = file
+		if Config().encrypt:
+			exitcode, real_filename, extra_headers["x-amz-meta-s3tools-gpgenc"] = gpg_encrypt(file)
+		response = s3.object_put_uri(real_filename, uri_final, extra_headers)
 		output("File '%s' stored as %s (%d bytes)" %
 			(file, uri_final, response["size"]))
 		if Config().acl_public:
 			output("Public URL of the object is: %s" %
 				(uri.public_url()))
+		if Config().encrypt and real_filename != file:
+			debug("Removing temporary encrypted file: %s" % real_filename)
+			os.remove(real_filename)
 
 def cmd_object_get(args):
 	s3 = S3(Config())
@@ -195,6 +203,9 @@ def cmd_object_get(args):
 	if not Config().force and os.path.exists(destination):
 		raise ParameterError("File %s already exists. Use --force to overwrite it" % destination)
 	response = s3.object_get_uri(uri, destination)
+	if response["headers"].has_key("x-amz-meta-s3tools-gpgenc"):
+		gpg_decrypt(destination, response["headers"]["x-amz-meta-s3tools-gpgenc"])
+		response["size"] = os.stat(destination)[6]
 	if destination != "-":
 		output("Object %s saved as '%s' (%d bytes)" %
 			(uri, destination, response["size"]))
@@ -210,6 +221,52 @@ def cmd_object_del(args):
 	response = s3.object_delete_uri(uri)
 	output("Object %s deleted" % uri)
 
+def resolve_list(lst, args):
+	retval = []
+	for item in lst:
+		retval.append(item % args)
+	return retval
+
+def gpg_command(command, passphrase = ""):
+	p_in, p_out = os.popen4(command)
+	if command.count("--passphrase-fd"):
+		p_in.write(passphrase+"\n")
+		p_in.flush()
+	for line in p_out:
+		info(line.strip())
+	p_pid, p_exitcode = os.wait()
+	return p_exitcode
+
+def gpg_encrypt(filename):
+	tmp_filename = Utils.mktmpfile()
+	args = {
+		"gpg_command" : cfg.gpg_command,
+		"passphrase_fd" : "0",
+		"input_file" : filename, 
+		"output_file" : tmp_filename,
+	}
+	info("Encrypting file %(input_file)s to %(output_file)s..." % args)
+	command = resolve_list(cfg.gpg_encrypt.split(" "), args)
+	code = gpg_command(command, cfg.gpg_passphrase)
+	return (code, tmp_filename, "gpg")
+
+def gpg_decrypt(filename, gpgenc_header = ""):
+	tmp_filename = Utils.mktmpfile(filename)
+	args = {
+		"gpg_command" : cfg.gpg_command,
+		"passphrase_fd" : "0",
+		"input_file" : filename, 
+		"output_file" : tmp_filename,
+	}
+	info("Decrypting file %(input_file)s to %(output_file)s..." % args)
+	command = resolve_list(cfg.gpg_decrypt.split(" "), args)
+	code = gpg_command(command, cfg.gpg_passphrase)
+	if code == 0:
+		debug("Renaming %s to %s" % (tmp_filename, filename))
+		os.unlink(filename)
+		os.rename(tmp_filename, filename)
+	return (code)
+
 def run_configure(config_file):
 	cfg = Config()
 	options = [
@@ -325,6 +382,7 @@ if __name__ == '__main__':
 	optparser.add_option("-c", "--config", dest="config", metavar="FILE", help="Config file name. Defaults to %default")
 	optparser.add_option(      "--dump-config", dest="dump_config", action="store_true", help="Dump current configuration after parsing config files and command line options and exit.")
 
+	optparser.add_option("-e", "--encrypt", dest="encrypt", action="store_true", help="Encrypt files before uploading to S3.")
 	optparser.add_option("-f", "--force", dest="force", action="store_true", help="Force overwrite and other dangerous operations.")
 	optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read by anyone.")
 
@@ -382,6 +440,11 @@ if __name__ == '__main__':
 			## Some Config() options are not settable from command line
 			pass
 
+	if cfg.encrypt and cfg.gpg_passphrase == "":
+		error("Encryption requested but no passphrase set in config file.")
+		error("Please re-run 's3cmd --configure' and supply it.")
+		sys.exit(1)
+
 	if options.dump_config:
 		cfg.dump_config(sys.stdout)
 		sys.exit(0)