LIBCLOUD-781 Introducing Container-as-a-Service cloud as an API base driver to libcloud

.ratignore

@@ -10,12 +10,14 @@ test/storage/fixtures/
 test/compute/fixtures/
 test/loadbalancer/fixtures/
 test/dns/fixtures/
+test/container/fixtures/
 coverage_html_report/
 .coverage
 .coveragerc
 libcloud/data/pricing.json
 libcloud/common/__init__.py
 libcloud/compute/__init__.py
+libcloud/container/__init__.py
 libcloud/storage/__init__.py
 libcloud/loadbalancer/__init__.py
 libcloud/dns/__init__.py

.travis.yml

@@ -8,7 +8,6 @@ python:
     - 3.4
     - 3.5
     - "pypy"
-sudo: false
 os:
   - linux
   # Note: OS X has been broken on Travis for a long time
@@ -17,15 +16,18 @@ env:
 
 matrix:
   fast_finish: true
-  allow_failures:
-    - os: osx
   include:
-    - python: 2.7
+    - sudo: required
+      python: 2.7
       env: ENV=lint
       before_script: TOX_ENV=lint
-    - python: 2.7
+    - sudo: required
+      python: 2.7
       env: ENV=docs
       before_script: TOX_ENV=docs-travis
+      before_install:
+        - sudo apt-get update -qq
+        - sudo apt-get install -y graphviz
       # Trigger ReadTheDocs build
       after_success: ./contrib/trigger_rtd_build.py 8284
 

CHANGES.rst

@@ -7,6 +7,11 @@ Changes with current version of Apache Libcloud
 General
 ~~~~~~~
 
+- Introduction of container based drivers for Docker, Rkt and Container-as-a-service
+  providers
+  (LIBCLOUD-781, GITHUB-666)
+  [Anthony Shaw]
+
 - Introduce a new ``libcloud.backup`` API for Backup as a Service projects and
   products.
   (GITHUB-621)
@@ -63,7 +68,7 @@ DNS
   [Wido den Hollander]
 
 Changes with Apache Libcloud 0.20.0
------------------------------------
+-------------------------------------------
 
 General
 ~~~~~~~

README.rst

@@ -34,6 +34,8 @@ Resources you can manage with Libcloud are divided into the following categories
   CloudFiles (``libcloud.storage.*``)
 * **Load Balancers** - Load Balancers as a Service, LBaaS (``libcloud.loadbalancer.*``)
 * **DNS** - DNS as a Service, DNSaaS (``libcloud.dns.*``)
+* **Container** - Container virtualization services (``libcloud.container.*``)
+
 
 Apache Libcloud is an Apache project, see <http://libcloud.apache.org> for
 more information.

contrib/generate_provider_feature_matrix_table.py

@@ -45,6 +45,11 @@
 from libcloud.dns.providers import DRIVERS as DNS_DRIVERS
 from libcloud.dns.types import Provider as DNSProvider
 
+from libcloud.container.base import ContainerDriver
+from libcloud.container.providers import get_driver as get_container_driver
+from libcloud.container.providers import DRIVERS as CONTAINER_DRIVERS
+from libcloud.container.types import Provider as ContainerProvider
+
 from libcloud.backup.base import BackupDriver
 from libcloud.backup.providers import get_driver as get_backup_driver
 from libcloud.backup.providers import DRIVERS as BACKUP_DRIVERS
@@ -94,6 +99,11 @@
     'dns': ['list_zones', 'list_records', 'iterate_zones', 'iterate_records',
             'create_zone', 'update_zone', 'create_record', 'update_record',
             'delete_zone', 'delete_record'],
+    'container': ['install_image', 'list_images', 'deploy_container',
+                  'get_container', 'start_container', 'stop_container',
+                  'restart_container', 'destroy_container', 'list_containers',
+                  'list_locations', 'create_cluster', 'destroy_cluster',
+                  'list_clusters'],
     'backup': ['get_supported_target_types', 'list_targets', 'create_target', 'create_target_from_node',
                'create_target_from_storage_container', 'update_target', 'delete_target', 'list_recovery_points',
                'recover_target', 'recover_target_out_of_place', 'list_target_jobs', 'create_target_job',
@@ -169,6 +179,21 @@
         'delete_zone': 'delete zone',
         'delete_record': 'delete record'
     },
+    'container': {
+        'install_image': 'install image',
+        'list_images': 'list images',
+        'deploy_container': 'deploy container',
+        'get_container': 'get container',
+        'list_containers': 'list containers',
+        'start_container': 'start container',
+        'stop_container': 'stop container',
+        'restart_container': 'restart container',
+        'destroy_container': 'destroy container',
+        'list_locations': 'list locations',
+        'create_cluster': 'create cluster',
+        'destroy_cluster': 'destroy cluster',
+        'list_clusters': 'list clusters'
+    },
     'backup': {
         'get_supported_target_types': 'get supported target types',
         'list_targets': 'list targets',
@@ -185,7 +210,7 @@
         'resume_target_job': 'resume target job',
         'suspend_target_job': 'suspend target job',
         'cancel_target_job': 'cancel target job'
-    },
+    }
 }
 
 IGNORED_PROVIDERS = [
@@ -229,6 +254,11 @@ def generate_providers_table(api):
         drivers = DNS_DRIVERS
         provider = DNSProvider
         get_driver_method = get_dns_driver
+    elif api == 'container':
+        driver = ContainerDriver
+        drivers = CONTAINER_DRIVERS
+        provider = ContainerProvider
+        get_driver_method = get_container_driver
     elif api == 'backup':
         driver = BackupDriver
         drivers = BACKUP_DRIVERS

docs/conf.py

@@ -44,7 +44,7 @@
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx',
-              'sphinx.ext.viewcode']
+              'sphinx.ext.viewcode', 'sphinx.ext.graphviz']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']

docs/container/_supported_methods.rst

@@ -0,0 +1,15 @@
+.. NOTE: This file has been generated automatically using generate_provider_feature_matrix_table.py script, don't manually edit it
+
+=================================== ============= =========== ================ ============= =============== ============== ================= ================= =============== ============== ============== =============== =============
+Provider                            install image list images deploy container get container start container stop container restart container destroy container list containers list locations create cluster destroy cluster list clusters
+=================================== ============= =========== ================ ============= =============== ============== ================= ================= =============== ============== ============== =============== =============
+`Docker`_                           yes           yes         yes              yes           yes             yes            yes               yes               yes             no             no             no              no           
+`Amazon Elastic Container Service`_ no            yes         yes              yes           yes             yes            yes               yes               yes             no             yes            yes             yes          
+`Joyent Triton`_                    yes           yes         yes              yes           yes             yes            yes               yes               yes             no             no             no              no           
+`Kubernetes`_                       no            no          yes              yes           no              no             no                yes               yes             no             yes            yes             yes          
+=================================== ============= =========== ================ ============= =============== ============== ================= ================= =============== ============== ============== =============== =============
+
+.. _`Docker`: http://docker.io
+.. _`Amazon Elastic Container Service`: https://aws.amazon.com/ecs/details/
+.. _`Joyent Triton`: http://joyent.com
+.. _`Kubernetes`: http://kubernetes.io

docs/container/_supported_providers.rst

@@ -0,0 +1,15 @@
+.. NOTE: This file has been generated automatically using generate_provider_feature_matrix_table.py script, don't manually edit it
+
+=================================== ============================================ ================= ============================================ ==================================
+Provider                            Documentation                                Provider constant Module                                       Class Name                        
+=================================== ============================================ ================= ============================================ ==================================
+`Docker`_                           :doc:`Click </container/drivers/docker>`     DOCKER            :mod:`libcloud.container.drivers.docker`     :class:`DockerContainerDriver`    
+`Amazon Elastic Container Service`_ :doc:`Click </container/drivers/ecs>`        ECS               :mod:`libcloud.container.drivers.ecs`        :class:`ElasticContainerDriver`   
+`Joyent Triton`_                    :doc:`Click </container/drivers/joyent>`     JOYENT            :mod:`libcloud.container.drivers.joyent`     :class:`JoyentContainerDriver`    
+`Kubernetes`_                       :doc:`Click </container/drivers/kubernetes>` KUBERNETES        :mod:`libcloud.container.drivers.kubernetes` :class:`KubernetesContainerDriver`
+=================================== ============================================ ================= ============================================ ==================================
+
+.. _`Docker`: http://docker.io
+.. _`Amazon Elastic Container Service`: https://aws.amazon.com/ecs/details/
+.. _`Joyent Triton`: http://joyent.com
+.. _`Kubernetes`: http://kubernetes.io

docs/container/api.rst

@@ -0,0 +1,19 @@
+:orphan:
+
+Container Base API
+==================
+
+.. autoclass:: libcloud.container.base.ContainerDriver
+    :members:
+
+.. autoclass:: libcloud.container.base.Container
+    :members:
+
+.. autoclass:: libcloud.container.base.ContainerImage
+    :members:
+
+.. autoclass:: libcloud.container.base.ContainerCluster
+    :members:
+
+.. autoclass:: libcloud.container.base.ClusterLocation
+    :members:

docs/container/drivers/docker.rst

@@ -0,0 +1,26 @@
+Docker Container Driver Documentation
+=====================================
+
+`Docker`_ containers wrap up a piece of software in a complete filesystem that contains everything it needs to run:
+code, runtime, system tools, system libraries – anything you can install on a server. This guarantees that it will always run the same,
+regardless of the environment it is running in.
+
+.. figure:: /_static/images/provider_logos/docker.png
+    :align: center
+    :width: 300
+    :target: http://docker.io/
+
+Instantiating the driver
+------------------------
+
+.. literalinclude:: /examples/container/docker/instantiate_driver.py
+   :language: python
+
+API Docs
+--------
+
+.. autoclass:: libcloud.container.drivers.docker.DockerContainerDriver
+    :members:
+    :inherited-members:
+
+.. _`Docker`: https://docker.io/

docs/container/drivers/ecs.rst

@@ -0,0 +1,58 @@
+Amazon Elastic Container Service Documentation
+==============================================
+
+Elastic Container Service is a container-as-a-service feature of `AWS`_.
+
+.. figure:: /_static/images/provider_logos/aws.png
+    :align: center
+    :width: 300
+    :target: http://aws.amazon.com/
+
+To provide API key access, you should apply one of the roles:
+* AmazonEC2ContainerServiceFullAccess
+* AmazonEC2ContainerServiceReadOnlyAccess
+
+Instantiating the driver
+------------------------
+
+.. literalinclude:: /examples/container/ecs/instantiate_driver.py
+   :language: python
+
+Deploying a container
+---------------------
+
+.. literalinclude:: /examples/container/ecs/deploy_container.py
+   :language: python
+
+Deploying a container from Docker Hub
+-------------------------------------
+
+Docker Hub Client :class:`~libcloud.container.utils.docker.HubClient` is a shared utility class for interfacing to the public Docker Hub Service.
+
+You can use this class for fetching images to deploy to services like ECS
+
+.. literalinclude:: /examples/container/docker_hub.py
+   :language: python
+
+Deploying a container from Amazon Elastic Container Registry (ECR)
+------------------------------------------------------------------
+
+Amazon ECR is a combination of the Docker Registry V2 API and a proprietary API. The ECS driver includes methods for talking to both APIs.
+
+Docker Registry API Client :class:`~libcloud.container.utils.docker.RegistryClient` is a shared utility class for interfacing to the public Docker Hub Service.
+
+You can use a factory method to generate an instance of RegsitryClient from the ECS driver. This will request a 12 hour token from the Amazon API and instantiate a :class:`~libcloud.container.utils.docker.RegistryClient`
+object with those credentials.
+
+.. literalinclude:: /examples/container/ecs/container_registry.py
+   :language: python
+
+API Docs
+--------
+
+.. autoclass:: libcloud.container.drivers.ecs.ElasticContainerDriver
+    :members:
+    :inherited-members:
+
+
+.. _`AWS`: https://aws.amazon.com/

docs/container/drivers/index.rst

@@ -0,0 +1,12 @@
+:orphan:
+
+Container Drivers Documentation
+===============================
+
+This chapter includes links to driver (provider) specific documentation pages.
+
+.. toctree::
+    :glob:
+    :maxdepth: 1
+
+    *

docs/container/drivers/joyent.rst

@@ -0,0 +1,64 @@
+Joyent Triton Container Driver Documentation
+============================================
+
+`Joyent Triton`_ is a Docker hosting service, provided by service provider Joyent.
+Docker-native tools and elastic hosts make deploying on Triton as easy as running Docker on your laptop.
+There is no special software to install or configure.
+Mix Docker containers with container-native Linux to extend the benefits of containerization to legacy applications and stateful services.
+
+.. figure:: /_static/images/provider_logos/triton.png
+    :align: center
+    :width: 300
+    :target: http://joyent.com/
+
+Instantiating the driver
+------------------------
+
+Download the script::
+
+    curl -O https://raw.githubusercontent.com/joyent/sdc-docker/master/tools/sdc-docker-setup.sh
+
+Now execute the script, substituting the correct values::
+
+    bash sdc-docker-setup.sh <CLOUDAPI_URL> <ACCOUNT_USERNAME> ~/.ssh/<PRIVATE_KEY_FILE>
+
+This should output something similar to the following::
+
+    Setting up Docker client for SDC using:
+        CloudAPI:        https://us-east-1.api.joyent.com
+        Account:         jill
+        Key:             /Users/localuser/.ssh/sdc-docker.id_rsa
+
+    If you have a pass phrase on your key, the openssl command will
+    prompt you for your pass phrase now and again later.
+
+    Verifying CloudAPI access.
+    CloudAPI access verified.
+
+    Generating client certificate from SSH private key.
+    writing RSA key
+    Wrote certificate files to /Users/localuser/.sdc/docker/jill
+
+    Get Docker host endpoint from cloudapi.
+    Docker service endpoint is: tcp://us-east-1.docker.joyent.com:2376
+
+    * * *
+    Success. Set your environment as follows:
+
+        export DOCKER_CERT_PATH=/Users/localuser/.sdc/docker/jill
+        export DOCKER_HOST=tcp://us-east-1.docker.joyent.com:2376
+        export DOCKER_CLIENT_TIMEOUT=300
+        export DOCKER_TLS_VERIFY=1
+
+.. literalinclude:: /examples/container/joyent/instantiate_driver.py
+   :language: python
+
+API Docs
+--------
+
+.. autoclass:: libcloud.container.drivers.joyent.JoyentContainerDriver
+    :members:
+    :inherited-members:
+
+
+.. _`Joyent Triton`: https://www.joyent.com/blog/understanding-triton-containers