Non-client library example of constructing a Signed URL (#1837)

* Humble beginnings

* Update

* Update

* Update

* Add test and README for generate_signed_urls.py

* Update error message

* Fix issues in CLI

* Fix region tag and address feedback.

* Add spacing

* Python 2 and 3 compatible string to hex used

* Fixed too long line

storage/signed_urls/README.rst

@@ -0,0 +1,97 @@
+.. This file is automatically generated. Do not edit this file directly.
+
+Google Cloud Storage Python Samples
+===============================================================================
+
+.. image:: https://gstatic.com/cloudssh/images/open-btn.png
+   :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/signed_urls/README.rst
+
+
+This directory contains samples for Google Cloud Storage. `Google Cloud Storage`_ allows world-wide storage and retrieval of any amount of data at any time.
+
+
+
+
+.. _Google Cloud Storage: https://cloud.google.com/storage/docs
+
+Setup
+-------------------------------------------------------------------------------
+
+
+Authentication
+++++++++++++++
+
+This sample requires you to have authentication setup. Refer to the
+`Authentication Getting Started Guide`_ for instructions on setting up
+credentials for applications.
+
+.. _Authentication Getting Started Guide:
+    https://cloud.google.com/docs/authentication/getting-started
+
+Install Dependencies
+++++++++++++++++++++
+
+#. Clone python-docs-samples and change directory to the sample directory you want to use.
+
+    .. code-block:: bash
+
+        $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git
+
+#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions.
+
+   .. _Python Development Environment Setup Guide:
+       https://cloud.google.com/python/setup
+
+#. Create a virtualenv. Samples are compatible with Python 2.7 and 3.4+.
+
+    .. code-block:: bash
+
+        $ virtualenv env
+        $ source env/bin/activate
+
+#. Install the dependencies needed to run the samples.
+
+    .. code-block:: bash
+
+        $ pip install -r requirements.txt
+
+.. _pip: https://pip.pypa.io/
+.. _virtualenv: https://virtualenv.pypa.io/
+
+Samples
+-------------------------------------------------------------------------------
+
+Generate Signed URLs in Python
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+.. image:: https://gstatic.com/cloudssh/images/open-btn.png
+   :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/signed_urls/generate_signed_urls.py,storage/signed_urls/README.rst
+
+
+
+
+To run this sample:
+
+.. code-block:: bash
+
+    $ python generate_signed_urls.py
+
+    usage: generate_signed_urls.py [-h]
+                                   service_account_file request_method bucket_name
+                                   object_name expiration
+
+    positional arguments:
+      service_account_file  Path to your Google service account.
+      request_method        A request method, e.g GET, POST.
+      bucket_name           Your Cloud Storage bucket name.
+      object_name           Your Cloud Storage object name.
+      expiration            Expiration Time.
+
+    optional arguments:
+      -h, --help            show this help message and exit
+
+
+
+
+
+.. _Google Cloud SDK: https://cloud.google.com/sdk/

storage/signed_urls/README.rst.in

@@ -0,0 +1,22 @@
+# This file is used to generate README.rst
+
+product:
+  name: Google Cloud Storage
+  short_name: Cloud Storage
+  url: https://cloud.google.com/storage/docs
+  description: >
+    `Google Cloud Storage`_ allows world-wide storage and retrieval of any
+    amount of data at any time.
+
+setup:
+- auth
+- install_deps
+
+samples:
+- name: Generate Signed URLs in Python
+  file: generate_signed_urls.py
+  show_help: true
+
+cloud_client_library: false
+
+folder: storage/signed_urls

storage/signed_urls/generate_signed_urls.py

@@ -0,0 +1,167 @@
+# Copyright 2018 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+
+"""This application demonstrates how to construct a Signed URL for objects in
+   Google Cloud Storage.
+
+For more information, see the README.md under /storage and the documentation
+at https://cloud.google.com/storage/docs/access-control/signing-urls-manually.
+"""
+
+# [START storage_signed_url_all]
+# [START storage_signed_url_dependencies]
+import binascii
+import collections
+import datetime
+import hashlib
+import sys
+
+# pip install six
+from six.moves.urllib.parse import quote
+
+# [START storage_signed_url_signer]
+# pip install google-auth
+from google.oauth2 import service_account
+
+# [END storage_signed_url_signer]
+# [END storage_signed_url_dependencies]
+
+
+def generate_signed_url(service_account_file, bucket_name, object_name,
+                        expiration, http_method='GET', query_parameters=None,
+                        headers=None):
+
+    if expiration > 604800:
+        print('Expiration Time can\'t be longer than 604800 seconds (7 days).')
+        sys.exit(1)
+
+    # [START storage_signed_url_canonical_uri]
+    escaped_object_name = quote(object_name, safe='')
+    canonical_uri = '/{}/{}'.format(bucket_name, escaped_object_name)
+    # [END storage_signed_url_canonical_uri]
+
+    # [START storage_signed_url_canonical_datetime]
+    datetime_now = datetime.datetime.utcnow()
+    request_timestamp = datetime_now.strftime('%Y%m%dT%H%M%SZ')
+    datestamp = datetime_now.strftime('%Y%m%d')
+    # [END storage_signed_url_canonical_datetime]
+
+    # [START storage_signed_url_credentials]
+    # [START storage_signed_url_signer]
+    google_credentials = service_account.Credentials.from_service_account_file(
+        service_account_file)
+    # [END storage_signed_url_signer]
+    client_email = google_credentials.service_account_email
+    credential_scope = '{}/auto/gcs/goog4_request'.format(datestamp)
+    credential = '{}/{}'.format(client_email, credential_scope)
+    # [END storage_signed_url_credentials]
+
+    if headers is None:
+        headers = dict()
+    # [START storage_signed_url_canonical_headers]
+    headers['host'] = 'storage.googleapis.com'
+
+    canonical_headers = ''
+    ordered_headers = collections.OrderedDict(sorted(headers.items()))
+    for k, v in ordered_headers.items():
+        lower_k = str(k).lower()
+        strip_v = str(v).lower()
+        canonical_headers += '{}:{}\n'.format(lower_k, strip_v)
+    # [END storage_signed_url_canonical_headers]
+
+    # [START storage_signed_url_signed_headers]
+    signed_headers = ''
+    for k, _ in ordered_headers.items():
+        lower_k = str(k).lower()
+        signed_headers += '{};'.format(lower_k)
+    signed_headers = signed_headers[:-1]  # remove trailing ';'
+    # [END storage_signed_url_signed_headers]
+
+    if query_parameters is None:
+        query_parameters = dict()
+    # [START storage_signed_url_canonical_query_parameters]
+    query_parameters['X-Goog-Algorithm'] = 'GOOG4-RSA-SHA256'
+    query_parameters['X-Goog-Credential'] = credential
+    query_parameters['X-Goog-Date'] = request_timestamp
+    query_parameters['X-Goog-Expires'] = expiration
+    query_parameters['X-Goog-SignedHeaders'] = signed_headers
+
+    canonical_query_string = ''
+    ordered_query_parameters = collections.OrderedDict(
+        sorted(query_parameters.items()))
+    for k, v in ordered_query_parameters.items():
+        encoded_k = quote(str(k), safe='')
+        encoded_v = quote(str(v), safe='')
+        canonical_query_string += '{}={}&'.format(encoded_k, encoded_v)
+    canonical_query_string = canonical_query_string[:-1]  # remove trailing ';'
+    # [END storage_signed_url_canonical_query_parameters]
+
+    # [START storage_signed_url_canonical_request]
+    canonical_request = '\n'.join([http_method,
+                                   canonical_uri,
+                                   canonical_query_string,
+                                   canonical_headers,
+                                   signed_headers,
+                                   'UNSIGNED-PAYLOAD'])
+    # [END storage_signed_url_canonical_request]
+
+    # [START storage_signed_url_hash]
+    canonical_request_hash = hashlib.sha256(
+        canonical_request.encode()).hexdigest()
+    # [END storage_signed_url_hash]
+
+    # [START storage_signed_url_string_to_sign]
+    string_to_sign = '\n'.join(['GOOG4-RSA-SHA256',
+                                request_timestamp,
+                                credential_scope,
+                                canonical_request_hash])
+    # [END storage_signed_url_string_to_sign]
+
+    # [START storage_signed_url_signer]
+    signature = binascii.hexlify(
+        google_credentials.signer.sign(string_to_sign)
+    ).decode()
+    # [END storage_signed_url_signer]
+
+    # [START storage_signed_url_construction]
+    host_name = 'https://storage.googleapis.com'
+    signed_url = '{}{}?{}&x-goog-signature={}'.format(host_name, canonical_uri,
+                                                      canonical_query_string,
+                                                      signature)
+    # [END storage_signed_url_construction]
+    return signed_url
+# [END storage_signed_url_all]
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('service_account_file',
+                        help='Path to your Google service account.')
+    parser.add_argument(
+        'request_method', help='A request method, e.g GET, POST.')
+    parser.add_argument('bucket_name', help='Your Cloud Storage bucket name.')
+    parser.add_argument('object_name', help='Your Cloud Storage object name.')
+    parser.add_argument('expiration', help='Expiration Time.')
+
+    args = parser.parse_args()
+    signed_url = generate_signed_url(
+        service_account_file=args.service_account_file,
+        http_method=args.request_method, bucket_name=args.bucket_name,
+        object_name=args.object_name, expiration=int(args.expiration))
+
+    print(signed_url)

storage/signed_urls/generate_signed_urls_test.py

@@ -0,0 +1,42 @@
+# Copyright 2018 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from google.cloud import storage
+import pytest
+import requests
+
+import generate_signed_urls
+
+BUCKET = os.environ['CLOUD_STORAGE_BUCKET']
+GOOGLE_APPLICATION_CREDENTIALS = os.environ['GOOGLE_APPLICATION_CREDENTIALS']
+
+
+@pytest.fixture
+def test_blob():
+    """Provides a pre-existing blob in the test bucket."""
+    bucket = storage.Client().bucket(BUCKET)
+    blob = bucket.blob('storage_snippets_test_sigil')
+    blob.upload_from_string('Hello, is it me you\'re looking for?')
+    return blob
+
+
+def test_generate_get_signed_url(test_blob, capsys):
+    get_signed_url = generate_signed_urls.generate_signed_url(
+        service_account_file=GOOGLE_APPLICATION_CREDENTIALS,
+        bucket_name=BUCKET, object_name=test_blob.name,
+        expiration=60)
+    response = requests.get(get_signed_url)
+    assert response.ok

storage/signed_urls/requirements.txt

@@ -0,0 +1,3 @@
+google-cloud-storage==1.13.0
+google-auth==1.5.1
+six==1.11.0