To use gcs-client in a project you will need to have Credentials to access intended Google Cloud Storage.
Credentials are generated in Google Developers Console in the Credentials section of the API Manager of the project. Recommended credentials file is JSON.
Once you have the credentials you can start using gcs_client to access your project.
import gcs_client
credentials_file = 'private_key.json'
project_name = 'project_name'
credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)
buckets = project.list()
print 'Buckets:\n\t- ','\n\t- '.join(map(str, buckets))
import gcs_client
credentials_file = 'private_key.json'
project_name = 'project_name'
credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)
bucket = project.create_bucket('my_new_bucket', location='EU')
print 'Bucket %s is located in %s with storage class %s' % (bucket, bucket.location,
bucket.storageClass)
import gcs_client
credentials_file = 'private_key.json'
project_name = 'project_name'
credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)
buckets = project.list()
# Delete one bucket but never the default bucket
default_bucket = project.default_bucket_name
filtered = filter(lambda b: b.name != default_bucket, buckets)
if filtered:
buckets[0].delete()
import gcs_client
credentials_file = 'private_key.json'
project_name = 'project_name'
credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)
buckets = project.list()
objects = buckets[0].list()
print 'Contents of bucket %s:' % bucket
if objects:
print '\t','\n\t'.join(map(lambda o: o.name + ' has %s bytes' % o.size, objects))
else:
print '\tThere are no objects'
import gcs_client
credentials = gcs_client.Credentials('private_key.json')
bucket = gcs_client.Bucket('bucket_name', credentials)
directory = 'var/log'
objects = bucket.list(directory)
print 'Contents of %s/%s:' % (bucket.name, directory)
if objects:
print '\t','\n\t'.join(map(lambda o: o.name + ' has %s bytes' % o.size, objects))
else:
print '\tThere are no objects'
We can list a bucket as if it were a directory by passing delimiter
optional argument on the list
call.
Following example implements tree
command for a bucket.
This is just for demonstration purposes, since it is not efficient because for each "directory" we find we make another request to the server to list its contents. It would be more efficient to request all the objects in one go and then rebuild the tree locally.
import gcs_client
def print_obj(obj, i, last):
if isinstance(obj, gcs_client.Prefix):
name = obj.prefix.split('/')[-2]
else:
name = obj.name.split('/')[-1]
print (' ' * i) + ('└──' if last else '├──'), name
def tree(objs, indent=0):
if indent == 0:
print '.'
for i in range(len(objs)):
obj = objs[i]
print_obj(obj, indent, i == len(objs) - 1)
if isinstance(obj, gcs_client.Prefix):
tree(obj.list(), indent+1)
credentials = gcs_client.Credentials('private_key.json')
bucket = gcs_client.Bucket('bucket_name', credentials)
print '$ tree', bucket.name
tree(bucket.list(delimiter='/'))
import gcs_client
credentials_file = 'private_key.json'
project_name = 'project_name'
credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)
bucket = project.list()[0]
objects = bucket.list()
if objects:
obj = objects[0]
print 'Deleting object %s' % obj
obj.delete()
import gcs_client
credentials_file = 'private_key.json'
project_name = 'project_name'
credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)
buckets = project.list()
objects = buckets[0].list()
if objects:
with objects[0].open() as obj:
print 'Contents of file %s are:\n' % obj.name, obj.read()
import gcs_client
credentials = gcs_client.Credentials('private_key.json')
bucket = gcs_client.Bucket('bucket_name', credentials)
chunksize = 4 * 1024 * 1024
with bucket.open('my_file', 'r', chunksize=chunksize) as obj:
print 'Contents of file %s are:\n' % obj.name, obj.read()
import gcs_client
credentials_file = 'private_key.json'
project_name = 'project_name'
credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)
bucket = project.list()[0]
with bucket.open('new_file.txt', 'w') as obj:
obj.write('Hello world\n')
with bucket.open('new_file.txt') as obj:
print obj.read()
All operations use retries with Truncated Exponential Backoff by default, but we can change default configuration.
import gcs_client
# Set default retry configuration using a RetryParams instance
new_retry_cfg = gcs_client.RetryParams(max_retries=10, initial_delay=0.5, max_backoff=8,
randomize=False)
gcs_client.RetryParams.set_default(new_retry_cfg)
# Set default retry configuration via params
gcs_client.RetryParams.set_default(max_retries=10, initial_delay=0.5, max_backoff=8,
randomize=False)
We may want to disable all retries for all instances that are using default retry configuration. Those that are using specific instance configurations will continue doing so.
import gcs_client
# Disable retry configuration
gcs_client.RetryParams.set_default(0)
We can set specific retry configuration for an instance. Important to notice that listed objects will inherit retry configuration from the object that did the listing.
import gcs_client
credentials_file = 'private_key.json'
project_name = 'project_name'
credentials = gcs_client.Credentials(credentials_file)
project = gcs_client.Project(project_name, credentials)
bucket = project.list()[0]
# Set bucket retry configuration
bucket.retry_params = gcs_client.RetryParams(max_retries=10, initial_delay=0.5, max_backoff=8,
randomize=False)
# Disable retries on the bucket
bucket.retry_params = None