Write a set of short scripts using the kamaki lib

Refs grnet#49

docs/developers/examples.rst

@@ -0,0 +1,149 @@
+Examples
+========
+
+This is a collection of python scripts that demonstrate the usage of the kamaki
+clients API library
+
+Initial steps
+-------------
+
+Initialize clients
+""""""""""""""""""
+
+Initialize the identity (`auth`), storage (`store`), `compute`,
+`network`, block storage (`volume`) and `image` clients, given an
+authentication URL and TOKEN.
+
+.. literalinclude:: scripts/000_init.py
+    :language: python
+    :linenos:
+    :lines: 48-
+
+Authentication URL and TOKEN from config file
+"""""""""""""""""""""""""""""""""""""""""""""
+
+Drag the URL and TOKEN information from the kamaki configuration file, using
+the "Config" class from kamaki CLI.
+
+.. literalinclude:: scripts/000_init.py
+    :language: python
+    :linenos:
+    :lines: 34-37
+
+.. note:: The cloud URL and TOKEN are stored under a cloud name. Kamaki can be
+    configured to `use multiple clouds <../setup.html#multiple-clouds>`_. It is
+    common practice to
+    `set the value of the default cloud <../setup.html#quick-setup>`_ using the
+    **global.default_cloud** configuration option. Here it was assumed that the
+    value **global.default_cloud** is the name of the preferred cloud
+
+Log HTTP
+""""""""
+
+Instruct kamaki to output the HTTP logs on the console.
+
+.. literalinclude:: scripts/000_init.py
+    :language: python
+    :linenos:
+    :lines: 42-45
+
+Containers and files
+--------------------
+
+Information on a remote object
+""""""""""""""""""""""""""""""
+
+List all objects in the storage_client's default container and ask user to pick
+one of them for more information.
+
+.. literalinclude:: scripts/001_file_info.py
+    :language: python
+    :linenos:
+    :lines: 48-
+
+Backup container
+""""""""""""""""
+
+Back up the contents of the default container to a new container.
+
+.. literalinclude:: scripts/002_container_backup.py
+    :language: python
+    :linenos:
+    :lines: 48-
+
+Empty and delete containers
+"""""""""""""""""""""""""""
+
+Delete all containers if their names start with "backup".
+
+.. literalinclude:: scripts/003_container_cleanup.py
+    :language: python
+    :linenos:
+    :lines: 48-
+
+.. note:: The "del_container" method will empty the container. The
+    "purge_container" method will destroy an empty container. If the container
+    is not empty, it cannot be destroyed.
+
+.. note:: The "try-finally" clause is used to preserve the original container
+    settings of the client (usually "pithos")
+
+Upload and Download
+"""""""""""""""""""
+
+Upload a local file
+
+.. literalinclude:: scripts/004_upload_files.py
+    :language: python
+    :linenos:
+    :lines: 56-
+
+Download a remote object as local file
+
+.. literalinclude:: scripts/005_download_files.py
+    :language: python
+    :linenos:
+    :lines: 56-
+
+.. note:: The _gen callback function is used to show upload/download progress.
+    It is optional. It must be a python generator, for example:
+
+    .. literalinclude:: scripts/004_upload_files.py
+        :language: python
+        :linenos:
+        :lines: 48-54
+
+Asynchronous upload of many files
+"""""""""""""""""""""""""""""""""
+
+Upload all files in a directory asynchronously
+
+.. literalinclude:: scripts/006_async_upload.py
+    :language: python
+    :linenos:
+    :lines: 48-
+
+Reassign container
+""""""""""""""""""
+
+Each resource is assigned to a project, where the resource quotas are defined.
+With this script, users are prompted to choose a project to assign the default
+container.
+
+.. literalinclude:: scripts/007_container_reassign.py
+    :language: python
+    :linenos:
+    :lines: 48-
+
+Download and stream in parallel
+"""""""""""""""""""""""""""""""
+
+Download an object in chunks. Stream them as they are being downloaded.
+
+.. literalinclude:: scripts/008_stream.py
+    :language: python
+    :linenos:
+    :lines: 48-
+
+.. note:: The ``kamaki.clients.SilentEvent`` class extends ``threading.Thread``
+    in order to simplify thread handling.

docs/developers/scripts/000_init.py

@@ -0,0 +1,70 @@
+# Copyright 2014 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.
+
+from kamaki.cli.config import Config
+
+cnf = Config()
+CLOUD = cnf.get('global', 'default_cloud')
+URL = cnf.get_cloud(CLOUD, 'url')
+TOKEN = cnf.get_cloud(CLOUD, 'token')
+
+
+from kamaki.cli import logger
+
+logger.add_stream_logger('kamaki.clients.recv', fmt='< %(message)s')
+logger.add_stream_logger('kamaki.clients.send', fmt='> %(message)s')
+
+
+from kamaki.clients import astakos, pithos, cyclades, image
+
+identity_client = astakos.AstakosClient(URL, TOKEN)
+
+pithosURL = identity_client.get_endpoint_url(pithos.PithosClient.service_type)
+storage_client = pithos.PithosClient(pithosURL, TOKEN)
+storage_client.account = identity_client.user_info['id']
+storage_client.container = 'pithos'
+
+computeURL = identity_client.get_endpoint_url(
+    cyclades.CycladesComputeClient.service_type)
+compute_client = cyclades.CycladesComputeClient(computeURL, TOKEN)
+
+networkURL = identity_client.get_endpoint_url(
+    cyclades.CycladesNetworkClient.service_type)
+network_client = cyclades.CycladesNetworkClient(networkURL, TOKEN)
+
+volumeURL = identity_client.get_endpoint_url(
+    cyclades.CycladesBlockStorageClient.service_type)
+volume_client = cyclades.CycladesBlockStorageClient(volumeURL, TOKEN)
+
+imageURL = identity_client.get_endpoint_url(image.ImageClient.service_type)
+image_client = image.ImageClient(imageURL, TOKEN)

docs/developers/scripts/001_file_info.py

@@ -0,0 +1,55 @@
+# Copyright 2014 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.
+
+from kamaki.cli.config import Config
+from kamaki.clients import astakos, pithos, ClientError
+
+cnf = Config()
+CLOUD = cnf.get('global', 'default_cloud')
+URL = cnf.get_cloud(CLOUD, 'url')
+TOKEN = cnf.get_cloud(CLOUD, 'token')
+identity_client = astakos.AstakosClient(URL, TOKEN)
+
+pithosURL = identity_client.get_endpoint_url(pithos.PithosClient.service_type)
+storage_client = pithos.PithosClient(pithosURL, TOKEN)
+storage_client.account = identity_client.user_info['id']
+storage_client.container = 'pithos'
+
+import json
+
+names = [o['name'] for o in storage_client.list_objects()]
+print 'Remote objects:\n\t', '\n\t'.join(names)
+
+pick = raw_input('Pick one: ')
+remote_object = storage_client.get_object_info(pick)
+print json.dumps(remote_object, indent=2)

docs/developers/scripts/002_container_backup.py

@@ -0,0 +1,59 @@
+# Copyright 2014 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.
+
+from kamaki.cli.config import Config
+from kamaki.clients import astakos, pithos, ClientError
+
+cnf = Config()
+CLOUD = cnf.get('global', 'default_cloud')
+URL = cnf.get_cloud(CLOUD, 'url')
+TOKEN = cnf.get_cloud(CLOUD, 'token')
+identity_client = astakos.AstakosClient(URL, TOKEN)
+
+pithosURL = identity_client.get_endpoint_url(pithos.PithosClient.service_type)
+storage_client = pithos.PithosClient(pithosURL, TOKEN)
+storage_client.account = identity_client.user_info['id']
+storage_client.container = 'pithos'
+
+import time
+
+backup_container = 'backup_%s' % time.time()
+storage_client.create_container(backup_container)
+
+for o in storage_client.list_objects():
+    try:
+        storage_client.copy_object(
+            storage_client.container, o['name'], backup_container)
+    except ClientError as ce:
+        print "Failed to backup object %s" % o['name']
+        print ce

docs/developers/scripts/003_container_cleanup.py

@@ -0,0 +1,56 @@
+# Copyright 2014 GRNET S.A. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#   1. Redistributions of source code must retain the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above
+#      copyright notice, this list of conditions and the following
+#      disclaimer in the documentation and/or other materials
+#      provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and
+# documentation are those of the authors and should not be
+# interpreted as representing official policies, either expressed
+# or implied, of GRNET S.A.
+
+from kamaki.cli.config import Config
+from kamaki.clients import astakos, pithos, ClientError
+
+cnf = Config()
+CLOUD = cnf.get('global', 'default_cloud')
+URL = cnf.get_cloud(CLOUD, 'url')
+TOKEN = cnf.get_cloud(CLOUD, 'token')
+identity_client = astakos.AstakosClient(URL, TOKEN)
+
+pithosURL = identity_client.get_endpoint_url(pithos.PithosClient.service_type)
+storage_client = pithos.PithosClient(pithosURL, TOKEN)
+storage_client.account = identity_client.user_info['id']
+storage_client.container = 'pithos'
+
+current_container = storage_client.container
+try:
+    for c in storage_client.list_containers():
+        if c['name'].startswith('backup_'):
+            storage_client.container = c['name']
+            storage_client.del_container(delimiter='/')
+            storage_client.purge_container()
+finally:
+    storage_client.container = current_container