Google Cloud Storage (GCS)

.gitignore

@@ -56,3 +56,9 @@ target/
 # vim
 *.swp
 *.swo
+
+# PyCharm
+.idea/
+
+# env files
+.env

README.rst

@@ -12,7 +12,7 @@ smart_open — utils for streaming large files in Python
 What?
 =====
 
-``smart_open`` is a Python 2 & Python 3 library for **efficient streaming of very large files** from/to storages such as S3, HDFS, WebHDFS, HTTP, HTTPS, SFTP, or local filesystem. It supports transparent, on-the-fly (de-)compression for a variety of different formats.
+``smart_open`` is a Python 2 & Python 3 library for **efficient streaming of very large files** from/to storages such as S3, GCS, HDFS, WebHDFS, HTTP, HTTPS, SFTP, or local filesystem. It supports transparent, on-the-fly (de-)compression for a variety of different formats.
 
 ``smart_open`` is a drop-in replacement for Python's built-in ``open()``: it can do anything ``open`` can (100% compatible, falls back to native ``open`` wherever possible), plus lots of nifty extra stuff on top.
 
@@ -80,6 +80,7 @@ Other examples of URLs that ``smart_open`` accepts::
     s3://my_bucket/my_key
     s3://my_key:my_secret@my_bucket/my_key
     s3://my_key:my_secret@my_server:my_port@my_bucket/my_key
+    gs://my_bucket/my_blob
     hdfs:///path/file
     hdfs://path/file
     webhdfs://host:port/path/file
@@ -174,6 +175,14 @@ More examples
     with open('s3://bucket/key.txt', 'wb', transport_params=transport_params) as fout:
         fout.write(b'here we stand')
 
+    # stream from GCS
+    for line in open('gs://my_bucket/my_file.txt'):
+        print(line)
+
+    # stream content *into* GCS (write mode):
+    with open('gs://my_bucket/my_file.txt', 'wb') as fout:
+        fout.write(b'hello world')
+
 Supported Compression Formats
 -----------------------------
 
@@ -212,6 +221,7 @@ Transport-specific Options
 - HTTP, HTTPS (read-only)
 - SSH, SCP and SFTP
 - WebHDFS
+- GCS
 
 Each option involves setting up its own set of parameters.
 For example, for accessing S3, you often need to set up authentication, like API keys or a profile name.

integration-tests/test_gcs.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+import io
+import os
+
+import google.cloud.storage
+
+import smart_open
+
+_GCS_BUCKET = os.environ.get('SO_GCS_BUCKET')
+_GCS_URL = 'gs://' + _GCS_BUCKET
+assert _GCS_BUCKET is not None, 'please set the SO_GCS_BUCKET environment variable'
+
+
+def initialize_bucket():
+    storage_client = google.cloud.storage.Client()
+    bucket = storage_client.get_bucket(_GCS_BUCKET)
+    blobs = bucket.list_blobs()
+    for blob in blobs:
+        blob.delete()
+
+
+def write_read(key, content, write_mode, read_mode, encoding=None, **kwargs):
+    with smart_open.open(key, write_mode, encoding=encoding, **kwargs) as fout:
+        fout.write(content)
+    with smart_open.open(key, read_mode, encoding=encoding, **kwargs) as fin:
+        actual = fin.read()
+    return actual
+
+
+def read_length_prefixed_messages(key, read_mode, encoding=None, **kwargs):
+    with smart_open.open(key, read_mode, encoding=encoding, **kwargs) as fin:
+        actual = b''
+        length_byte = fin.read(1);
+        while len(length_byte):
+            actual += length_byte
+            msg = fin.read(ord(length_byte))
+            actual += msg
+            length_byte = fin.read(1)
+    return actual
+
+
+def test_gcs_readwrite_text(benchmark):
+    initialize_bucket()
+
+    key = _GCS_URL + '/sanity.txt'
+    text = 'с гранатою в кармане, с чекою в руке'
+    actual = benchmark(write_read, key, text, 'w', 'r', 'utf-8')
+    assert actual == text
+
+
+def test_gcs_readwrite_text_gzip(benchmark):
+    initialize_bucket()
+
+    key = _GCS_URL + '/sanity.txt.gz'
+    text = 'не чайки здесь запели на знакомом языке'
+    actual = benchmark(write_read, key, text, 'w', 'r', 'utf-8')
+    assert actual == text
+
+
+def test_gcs_readwrite_binary(benchmark):
+    initialize_bucket()
+
+    key = _GCS_URL + '/sanity.txt'
+    binary = b'this is a test'
+    actual = benchmark(write_read, key, binary, 'wb', 'rb')
+    assert actual == binary
+
+
+def test_gcs_readwrite_binary_gzip(benchmark):
+    initialize_bucket()
+
+    key = _GCS_URL + '/sanity.txt.gz'
+    binary = b'this is a test'
+    actual = benchmark(write_read, key, binary, 'wb', 'rb')
+    assert actual == binary
+
+
+def test_gcs_performance(benchmark):
+    initialize_bucket()
+
+    one_megabyte = io.BytesIO()
+    for _ in range(1024*128):
+        one_megabyte.write(b'01234567')
+    one_megabyte = one_megabyte.getvalue()
+
+    key = _GCS_URL + '/performance.txt'
+    actual = benchmark(write_read, key, one_megabyte, 'wb', 'rb')
+    assert actual == one_megabyte
+
+
+def test_gcs_performance_gz(benchmark):
+    initialize_bucket()
+
+    one_megabyte = io.BytesIO()
+    for _ in range(1024*128):
+        one_megabyte.write(b'01234567')
+    one_megabyte = one_megabyte.getvalue()
+
+    key = _GCS_URL + '/performance.txt.gz'
+    actual = benchmark(write_read, key, one_megabyte, 'wb', 'rb')
+    assert actual == one_megabyte
+
+
+def test_gcs_performance_small_reads(benchmark):
+    initialize_bucket()
+
+    ONE_MIB = 1024**2
+    one_megabyte_of_msgs = io.BytesIO()
+    msg = b'\x0f' + b'0123456789abcde' # a length-prefixed "message"
+    for _ in range(0, ONE_MIB, len(msg)):
+            one_megabyte_of_msgs.write(msg)
+    one_megabyte_of_msgs = one_megabyte_of_msgs.getvalue()
+
+    key = _GCS_URL + '/many_reads_performance.bin'
+
+    with smart_open.open(key, 'wb') as fout:
+        fout.write(one_megabyte_of_msgs)
+
+    actual = benchmark(read_length_prefixed_messages, key, 'rb', buffering=ONE_MIB)
+    assert actual == one_megabyte_of_msgs

setup.py

@@ -59,14 +59,15 @@ def read(fname):
     'boto >= 2.32',
     'requests',
     'boto3',
+    'google-cloud-storage',
 ]
 if sys.version_info[0] == 2:
     install_requires.append('bz2file')
 
 setup(
     name='smart_open',
     version=__version__,
-    description='Utils for streaming large files (S3, HDFS, gzip, bz2...)',
+    description='Utils for streaming large files (S3, HDFS, GCS, gzip, bz2...)',
     long_description=read('README.rst'),
 
     packages=find_packages(),
@@ -82,7 +83,7 @@ def read(fname):
     url='https://github.com/piskvorky/smart_open',
     download_url='http://pypi.python.org/pypi/smart_open',
 
-    keywords='file streaming, s3, hdfs',
+    keywords='file streaming, s3, hdfs, gcs',
 
     license='MIT',
     platforms='any',