Permalink
Browse files

Created an example of GCS signed URLs in Python using requests and Py…

…Crypto.
  • Loading branch information...
1 parent 970cbbb commit 828486a99e34d38fc3ccbb434899284c8b069044 @jterrace jterrace committed Jan 10, 2013
Showing with 228 additions and 0 deletions.
  1. +2 −0 .gitignore
  2. +31 −0 README.md
  3. +17 −0 conf.example.py
  4. +176 −0 gcs-signed-url-example.py
  5. +2 −0 requirements.txt
View
@@ -0,0 +1,2 @@
+conf.py
+privatekey.der
View
@@ -0,0 +1,31 @@
+# Google Cloud Storage Signed URLs Example
+
+This script is an example of using a service account's private key to create
+signatures required for clients to access Google Cloud Storage using [signed URL
+authentication](https://developers.google.com/storage/docs/accesscontrol#Signed-URLs).
+
+## Required Dependencies
+
+The following third-party Python modules are required:
+
+ * [requests](http://docs.python-requests.org/en/latest/)
+ * [PyCrypto](https://www.dlitz.net/software/pycrypto/)
+
+The easiest way to install the dependencies is to run:
+
+ pip install -r requirements.txt
+
+## Configuration
+
+The `conf.example.py` file must be copied to `conf.py` and the variables in the
+file must be filled in. See the example file for explanation of each variable.
+
+## Example Flow
+
+The example script's flow is as follows:
+
+ * Generates a signature for a PUT request and uses it to upload a new file.
+ * Generates a signature for a GET request and uses it to download the new
+ file that was just uploaded.
+ * Generates a signature for a DELETE request and uses it to delete the file
+ that was created.
View
@@ -0,0 +1,17 @@
+import os.path
+
+# The email address for your GCS service account being used for signatures.
+SERVICE_ACCOUNT_EMAIL = ('abcdef1234567890@developer.gserviceaccount.com')
+
+# Bucket name to use for writing example file.
+BUCKET_NAME = 'bucket-name'
+# Object name to use for writing example file.
+OBJECT_NAME = 'object.txt'
+
+# Set this to the path of your service account private key file, in DER format.
+#
+# Given a GCS key in pkcs12 format, convert it to PEM using this command:
+# openssl pkcs12 -in path/to/key.p12 -nodes -nocerts > path/to/key.pem
+# Given a GCS key in PEM format, convert it to DER format using this command:
+# openssl rsa -in privatekey.pem -inform PEM -out privatekey.der -outform DER
+PRIVATE_KEY_PATH = os.path.join(os.path.dirname(__file__), 'privatekey.der')
View
@@ -0,0 +1,176 @@
+"""test docstring."""
+
+import base64
+import datetime
+import md5
+import sys
+import time
+
+import Crypto.Hash.SHA256 as SHA256
+import Crypto.PublicKey.RSA as RSA
+import Crypto.Signature.PKCS1_v1_5 as PKCS1_v1_5
+import requests
+
+try:
+ import conf
+except ImportError:
+ sys.exit('Configuration module not found. You must create a conf.py file. '
+ 'See the example in conf.example.py.')
+
+# The Google Cloud Storage API endpoint. You should not need to change this.
+GCS_API_ENDPOINT = 'https://storage.googleapis.com'
+
+
+class CloudStorageURLSigner(object):
+ """Contains methods for generating signed URLs for Google Cloud Storage."""
+
+ def __init__(self, key, client_id_email, gcs_api_endpoint, expiration=None,
+ session=None):
+ """Creates a CloudStorageURLSigner that can be used to access signed URLs.
+
+ Args:
+ key: A PyCrypto private key.
+ client_id_email: GCS service account email.
+ gcs_api_endpoint: Base URL for GCS API.
+ expiration: An instance of datetime.datetime containing the time when the
+ signed URL should expire.
+ session: A requests.session.Session to use for issuing requests. If not
+ supplied, a new session is created.
+ """
+ self.key = key
+ self.client_id_email = client_id_email
+ self.gcs_api_endpoint = gcs_api_endpoint
+
+ self.expiration = expiration
+ if self.expiration is None:
+ self.expiration = datetime.datetime.now() + datetime.timedelta(days=1)
+ self.expiration = int(time.mktime(self.expiration.timetuple()))
+
+ self.session = session
bensonk
bensonk Jan 11, 2013 Member

A potentially more pythonic way to do this would be something along these lines:

self.session = session or requests.Session()

jterrace
jterrace Jan 11, 2013 Collaborator

Done.

+ if self.session is None:
+ self.session = requests.Session()
+
+ def _Base64Sign(self, plaintext):
+ """Signs and returns a base64-encoded SHA256 digest."""
+ shahash = SHA256.new(plaintext)
+ signer = PKCS1_v1_5.new(self.key)
+ signature_bytes = signer.sign(shahash)
+ return base64.b64encode(signature_bytes)
+
+ def _MakeSignatureString(self, verb, path, content_md5, content_type):
+ """Creates the signature string for signing according to GCS docs."""
+ signature_string = ('%(verb)s\n'
+ '%(content_md5)s\n'
+ '%(content_type)s\n'
+ '%(expiration)s\n'
+ '%(resource)s')
+ return signature_string % {'verb': verb,
bensonk
bensonk Jan 11, 2013 Member

You might want to consider using locals() here (there are pros and cons) and/or switching to the more modern str.format() method rather than the old style % syntax.

jterrace
jterrace Jan 11, 2013 Collaborator

Switched to str.format. Agreeed it's much nicer than % syntax. I didn't use locals() though because it feels dirty :)

+ 'content_md5': content_md5,
+ 'content_type': content_type,
+ 'expiration': self.expiration,
+ 'resource': path}
+
+ def _MakeUrl(self, verb, path, content_type='', content_md5=''):
+ """Forms and returns the full signed URL to access GCS."""
+ base_url = '%s%s' % (self.gcs_api_endpoint, path)
+ signature_string = self._MakeSignatureString(verb, path, content_md5,
+ content_type)
+ signature_signed = self._Base64Sign(signature_string)
+ query_params = {'GoogleAccessId': self.client_id_email,
+ 'Expires': str(self.expiration),
+ 'Signature': signature_signed}
+ return base_url, query_params
+
+ def Get(self, path):
+ """Performs a GET request.
+
+ Args:
+ path: The relative API path to access, e.g. '/bucket/object'.
+
+ Returns:
+ An instance of requests.Response containing the HTTP response.
+ """
+ base_url, query_params = self._MakeUrl('GET', path)
+ return self.session.get(base_url, params=query_params)
+
+ def Put(self, path, content_type, data):
+ """Performs a PUT request.
+
+ Args:
+ path: The relative API path to access, e.g. '/bucket/object'.
+ content_type: The content type to assign to the upload.
+ data: The file data to upload to the new file.
+
+ Returns:
+ An instance of requests.Response containing the HTTP response.
+ """
+ md5_digest = base64.b64encode(md5.new(data).digest())
+ base_url, query_params = self._MakeUrl('PUT', path, content_type,
+ md5_digest)
+ headers = {}
+ headers['Content-Type'] = content_type
+ headers['Content-Length'] = str(len(data))
+ headers['Content-MD5'] = md5_digest
+ return self.session.put(base_url, params=query_params, headers=headers,
+ data=data)
+
+ def Delete(self, path):
+ """Performs a DELETE request.
+
+ Args:
+ path: The relative API path to access, e.g. '/bucket/object'.
+
+ Returns:
+ An instance of requests.Response containing the HTTP response.
+ """
+ base_url, query_params = self._MakeUrl('DELETE', path)
+ return self.session.delete(base_url, params=query_params)
+
+
+def ProcessResponse(r, expected_status=200):
+ """Prints request and response information and checks for desired return code.
+
+ Args:
+ r: A requests.Response object.
+ expected_status: The expected HTTP status code.
+
+ Raises:
+ SystemExit if the response code doesn't match expected_status.
+ """
+ print '--- Request ---'
+ print r.request.full_url
+ for header, value in r.request.headers.iteritems():
+ print '%s: %s' % (header, value)
+ print '---------------'
+ print '--- Response (Status %s) ---' % r.status_code
+ print r.content
+ print '-----------------------------'
+ print
+ if r.status_code != expected_status:
+ sys.exit('Exiting due to receiving %d status code when expecting %d.'
+ % (r.status_code, expected_status))
+
+
+def main():
+ private_key = RSA.importKey(open(conf.PRIVATE_KEY_PATH, 'rb').read())
lotten
lotten Jan 11, 2013

Would it make sense to catch a possible exception from open() ? (since you're also catching the conf import above...)

bensonk
bensonk Jan 11, 2013 Member

That seems pretty reasonable to me. If the user is expected to fill out conf.py and provide a private key, it's reasonable to produce equally friendly messages if either thing is missing.

jterrace
jterrace Jan 11, 2013 Collaborator

Done. Thanks to both of you for taking a look!

+ signer = CloudStorageURLSigner(private_key, conf.SERVICE_ACCOUNT_EMAIL,
+ GCS_API_ENDPOINT)
+
+ file_path = '/%s/%s' % (conf.BUCKET_NAME, conf.OBJECT_NAME)
+
+ print 'Creating file...'
+ print '================'
+ r = signer.Put(file_path, 'text/plain', 'blah blah')
+ ProcessResponse(r)
+ print 'Retrieving file...'
+ print '=================='
+ r = signer.Get(file_path)
+ ProcessResponse(r)
+ print 'Deleting file...'
+ print '================'
+ r = signer.Delete(file_path)
+ ProcessResponse(r, expected_status=204)
+ print 'Done.'
+
+if __name__ == '__main__':
+ main()
View
@@ -0,0 +1,2 @@
+requests
+PyCrypto

5 comments on commit 828486a

@bensonk
Member

LGTM. You might consider looking at http://stopwritingramblingcommitmessages.com/. :-)

@jterrace
Collaborator

Meh, I don't subscribe to Linus's arbitrary 50-character summary cutoff dogma. I usually justify my commits if they're long, but didn't seem necessary for a 76-wide line.

@bensonk
Member
@jterrace
Collaborator

Yeah, it makes me really angry that they do that, especially when they have the room for more characters. It's an awful UI decision, imo.

@palladius

I was actually educated in a previous role to write commit messages as an ABSTRACT of max 80 lines on first line, then an empty line, then the long explaination. I would adhere to it. Maybe we could work on a policy regarding commit messages with Marc?

Please sign in to comment.