[google compute] LIBCLOUD-657 add support for JSON private key format

docs/compute/drivers/gce.rst

@@ -37,7 +37,7 @@ Which one should I use?
 
 * If you are running your code on an instance inside Google Compute Engine,
   the GCE driver will consult the internal metadata service to obtain an
-  authorization token. The only parameter required for this type of
+  authorization token. The only value required for this type of
   authorization is your Project ID.
 
 Once you have set up the authentication as described below, you pass the
@@ -47,19 +47,29 @@ authentication information to the driver as described in `Examples`_
 Service Account
 ~~~~~~~~~~~~~~~
 
-To set up Service Account authentication:
+To set up Service Account authentication, you will need to download the
+corresponding private key file in either the new JSON (preferred) format, or
+the legacy P12 format.
 
 1. Follow the instructions at
    https://developers.google.com/console/help/new/#serviceaccounts
-   to create and download a PKCS-12 private key.
-2. Convert the PKCS-12 private key to a .pem file using the following:
-   ``openssl pkcs12 -in YOURPRIVKEY.p12 -nodes -nocerts
-   | openssl rsa -out PRIV.pem``
-3. Move the .pem file to a safe location
-4. You will need the Service Account's "Email Address" and the path to the
-   .pem file for authentication.
-5. You will also need your "Project ID" which can be found by clicking on the
-   "Overview" link on the left sidebar.
+   to create and download the private key.
+
+   a. If you opt for the new preferred JSON format, download the file and
+      save it to a secure location.
+
+   b. If you opt to use the legacy P12 format:
+
+      Convert the private key to a .pem file using the following:
+      ``openssl pkcs12 -in YOURPRIVKEY.p12 -nodes -nocerts
+      | openssl rsa -out PRIV.pem``
+
+      Move the .pem file to a safe location
+
+2. You will need the Service Account's "Email Address" and the path to the
+   key file for authentication.
+3. You will also need your "Project ID" (a string, not a numerical value) that
+   can be found by clicking on the "Overview" link on the left sidebar.
 
 Installed Application
 ~~~~~~~~~~~~~~~~~~~~~
@@ -72,8 +82,8 @@ To set up Installed Account authentication:
 4. Click on "Credentials" then "Create New Client ID"
 5. Select "Installed application" and "Other" then click "Create Client ID"
 6. For authentication, you will need the "Client ID" and the "Client Secret"
-7. You will also need your "Project ID" which can be found by clicking on the
-   "Overview" link on the left sidebar.
+7. You will also need your "Project ID" (a string, not a numerical value) that
+   can be found by clicking on the "Overview" link on the left sidebar.
 
 Internal Authentication
 ~~~~~~~~~~~~~~~~~~~~~~~

docs/examples/compute/gce/gce_internal_auth.py

@@ -2,8 +2,11 @@
 from libcloud.compute.providers import get_driver
 
 # This example assumes you are running on an instance within Google
-# Compute Engine. As such, the only parameter you need to specify is
+# Compute Engine. As such, the only value you need to specify is
 # the Project ID. The GCE driver will the consult GCE's internal
 # metadata service for an authorization token.
+#
+# You must still place placeholder empty strings for user_id / key
+# due to the nature of the driver's __init__() params.
 ComputeEngine = get_driver(Provider.GCE)
-driver = ComputeEngine(project='your_project_id')
+driver = ComputeEngine('', '', project='your_project_id')

docs/examples/compute/gce/gce_service_account.py

@@ -2,5 +2,7 @@
 from libcloud.compute.providers import get_driver
 
 ComputeEngine = get_driver(Provider.GCE)
+# Note that the 'PEM file' argument can either be the JSON format or
+# the P12 format.
 driver = ComputeEngine('your_service_account_email', 'path_to_pem_file',
                        project='your_project_id')

libcloud/common/google.py

@@ -79,7 +79,7 @@
 import sys
 
 from libcloud.utils.connection import get_response_object
-from libcloud.utils.py3 import httplib, urlencode, urlparse, PY3
+from libcloud.utils.py3 import b, httplib, urlencode, urlparse, PY3
 from libcloud.common.base import (ConnectionUserAndKey, JsonResponse,
                                   PollingConnection)
 from libcloud.common.types import (ProviderError,
@@ -345,7 +345,12 @@ def _token_request(self, request_body):
         """
         data = urlencode(request_body)
         now = self._now()
-        response = self.request('/o/oauth2/token', method='POST', data=data)
+        try:
+            response = self.request('/o/oauth2/token', method='POST',
+                                    data=data)
+        except AttributeError:
+            raise GoogleAuthError('Invalid authorization response, please '
+                                  'check your credentials.')
         token_info = response.object
         if 'expires_in' in token_info:
             expire_time = now + datetime.timedelta(
@@ -390,12 +395,12 @@ def get_code(self):
         data = urlencode(auth_params)
 
         url = 'https://%s%s?%s' % (self.host, self.auth_path, data)
-        print('Please Go to the following URL and sign in:')
+        print('\nPlease Go to the following URL and sign in:')
         print(url)
         if PY3:
-            code = input('Enter Code:')
+            code = input('Enter Code: ')
         else:
-            code = raw_input('Enter Code:')
+            code = raw_input('Enter Code: ')
         return code
 
     def get_new_token(self):
@@ -458,11 +463,21 @@ def __init__(self, user_id, key, *args, **kwargs):
             raise GoogleAuthError('PyCrypto library required for '
                                   'Service Account Authentication.')
         # Check to see if 'key' is a file and read the file if it is.
-        keypath = os.path.expanduser(key)
-        is_file_path = os.path.exists(keypath) and os.path.isfile(keypath)
-        if is_file_path:
+        if key.find("PRIVATE KEY---") == -1:
+            # key is a file
+            keypath = os.path.expanduser(key)
+            is_file_path = os.path.exists(keypath) and os.path.isfile(keypath)
+            if not is_file_path:
+                raise ValueError("Missing (or not readable) key "
+                                 "file: '%s'" % key)
             with open(keypath, 'r') as f:
-                key = f.read()
+                contents = f.read()
+            try:
+                key = json.loads(contents)
+                key = key['private_key']
+            except ValueError:
+                key = contents
+
         super(GoogleServiceAcctAuthConnection, self).__init__(
             user_id, key, *args, **kwargs)
 
@@ -475,26 +490,26 @@ def get_new_token(self):
         """
         # The header is always the same
         header = {'alg': 'RS256', 'typ': 'JWT'}
-        header_enc = base64.urlsafe_b64encode(json.dumps(header))
+        header_enc = base64.urlsafe_b64encode(b(json.dumps(header)))
 
         # Construct a claim set
         claim_set = {'iss': self.user_id,
                      'scope': self.scopes,
                      'aud': 'https://accounts.google.com/o/oauth2/token',
                      'exp': int(time.time()) + 3600,
                      'iat': int(time.time())}
-        claim_set_enc = base64.urlsafe_b64encode(json.dumps(claim_set))
+        claim_set_enc = base64.urlsafe_b64encode(b(json.dumps(claim_set)))
 
         # The message contains both the header and claim set
-        message = '%s.%s' % (header_enc, claim_set_enc)
+        message = b'.'.join((header_enc, claim_set_enc))
         # Then the message is signed using the key supplied
         key = RSA.importKey(self.key)
         hash_func = SHA256.new(message)
         signer = PKCS1_v1_5.new(key)
         signature = base64.urlsafe_b64encode(signer.sign(hash_func))
 
         # Finally the message and signature are sent to get a token
-        jwt = '%s.%s' % (message, signature)
+        jwt = b'.'.join((message, signature))
         request = {'grant_type': 'urn:ietf:params:oauth:grant-type:jwt-bearer',
                    'assertion': jwt}
 

libcloud/test/common/fixtures/google/pkey.json

@@ -0,0 +1,7 @@
+{
+  "private_key_id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxx\n-----END PRIVATE KEY-----\n",
+  "client_email": "foo@developer.gserviceaccount.com",
+  "client_id": "foo.apps.googleusercontent.com",
+  "type": "service_account"
+}

libcloud/test/common/fixtures/google/pkey.pem

@@ -0,0 +1,15 @@
+-----BEGIN RSA PRIVATE KEY-----
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+-----END RSA PRIVATE KEY-----

libcloud/test/common/test_google.py

@@ -18,6 +18,7 @@
 import datetime
 import sys
 import unittest
+import os
 
 try:
     import simplejson as json
@@ -33,7 +34,7 @@
                                     GoogleServiceAcctAuthConnection,
                                     GoogleGCEServiceAcctAuthConnection,
                                     GoogleBaseConnection)
-from libcloud.test.secrets import GCE_PARAMS
+
 
 # Skip some tests if PyCrypto is unavailable
 try:
@@ -42,6 +43,21 @@
     SHA256 = None
 
 
+SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))
+PEM_KEY = os.path.join(SCRIPT_PATH, "fixtures", "google", "pkey.pem")
+JSON_KEY = os.path.join(SCRIPT_PATH, "fixtures", "google", "pkey.json")
+with open(JSON_KEY, 'r') as f:
+    KEY_STR = json.loads(f.read())['private_key']
+
+
+GCE_PARAMS = ('email@developer.gserviceaccount.com', 'key')
+GCE_PARAMS_PEM_KEY = ('email@developer.gserviceaccount.com', PEM_KEY)
+GCE_PARAMS_JSON_KEY = ('email@developer.gserviceaccount.com', JSON_KEY)
+GCE_PARAMS_KEY = ('email@developer.gserviceaccount.com', KEY_STR)
+GCE_PARAMS_IA = ('client_id', 'client_secret')
+GCE_PARAMS_GCE = ('foo', 'bar')
+
+
 class MockJsonResponse(object):
     def __init__(self, body):
         self.object = body
@@ -146,17 +162,25 @@ def test_auth_type(self):
 
         if SHA256:
             kwargs['auth_type'] = 'SA'
-            conn1 = GoogleBaseConnection(*GCE_PARAMS, **kwargs)
+            conn1 = GoogleBaseConnection(*GCE_PARAMS_PEM_KEY, **kwargs)
+            self.assertTrue(isinstance(conn1.auth_conn,
+                                       GoogleServiceAcctAuthConnection))
+
+            conn1 = GoogleBaseConnection(*GCE_PARAMS_JSON_KEY, **kwargs)
+            self.assertTrue(isinstance(conn1.auth_conn,
+                                       GoogleServiceAcctAuthConnection))
+
+            conn1 = GoogleBaseConnection(*GCE_PARAMS_KEY, **kwargs)
             self.assertTrue(isinstance(conn1.auth_conn,
                                        GoogleServiceAcctAuthConnection))
 
         kwargs['auth_type'] = 'IA'
-        conn2 = GoogleBaseConnection(*GCE_PARAMS, **kwargs)
+        conn2 = GoogleBaseConnection(*GCE_PARAMS_IA, **kwargs)
         self.assertTrue(isinstance(conn2.auth_conn,
                                    GoogleInstalledAppAuthConnection))
 
         kwargs['auth_type'] = 'GCE'
-        conn3 = GoogleBaseConnection(*GCE_PARAMS, **kwargs)
+        conn3 = GoogleBaseConnection(*GCE_PARAMS_GCE, **kwargs)
         self.assertTrue(isinstance(conn3.auth_conn,
                                    GoogleGCEServiceAcctAuthConnection))
 

libcloud/test/compute/test_gce.py

@@ -85,7 +85,7 @@ def test_get_object_by_kind(self):
         obj = self.driver._get_object_by_kind(
             'https://www.googleapis.com/compute/v1/projects/project_name/'
             'global/targetHttpProxies/web-proxy')
-        self.assertEquals(obj.name, 'web-proxy')
+        self.assertEqual(obj.name, 'web-proxy')
 
     def test_get_region_from_zone(self):
         zone1 = self.driver.ex_get_zone('us-central1-a')