Merge pull request #301 from dutsii/docker_integration

Docker integration

dev-requirements.txt

@@ -12,3 +12,4 @@ yapf==0.27.0
 -r snmp-requirements.txt
 -r xena-requirements.txt
 -r graph-requirements.txt
+-r docker-requirements.txt

doc/configuration.rst

@@ -545,6 +545,37 @@ them to the internal environment.
 Used by:
   - potentially all drivers
 
+DockerDaemon
+~~~~~~~~~~~~
+A DockerDaemon describes where to contact a docker daemon process.
+DockerDaemon also participates in managing `NetworkService` instances
+created through interaction with that daemon.
+
+.. code-block:: yaml
+
+   DockerDaemon:
+     docker_daemon_url: 'unix://var/run/docker.sock'
+
+The example describes a docker daemon accessible via the
+'/var/run/docker.sock' unix socket. When used by a `DockerDriver`, the
+`DockerDriver` will first create a docker container which the
+DockerDaemon resource will subsequently use to create one/more
+`NetworkService` instances - as specified by `DockerDriver` configuration.
+Each `NetworkService` instance corresponds to a network service running inside
+the container.
+
+Moreover, DockerDaemon will remove any hanging containers if
+DockerDaemon is used several times in a row - as is the case when
+executing test suites. Normally `DockerDriver` - when deactivated -
+cleans up the created docker container; programming errors, keyboard
+interrupts or unix kill signals may lead to hanging containers, however;
+therefore auto-cleanup is important.
+
+- docker_daemon_url (str): The url of the daemon to use for this target.
+
+Used by:
+  - `DockerDriver`_
+
 udev Matching
 ~~~~~~~~~~~~~
 udev matching allows labgrid to identify resources via their udev properties.
@@ -1488,6 +1519,42 @@ Binds to:
 The driver is supposed to work with all Xena products from the "Valkyrie Layer 2-3 Test platform"
 Currently tested on a `XenaCompact` chassis equipped with a `1 GE test module`.
 
+DockerDriver
+~~~~~~~~~~~~
+A DockerDriver binds to a `DockerDaemon` and is used to create and control one
+docker container.
+
+| The driver uses the docker python module to interact with the docker daemon.
+| For more information on the parameters see:
+| https://docker-py.readthedocs.io/en/stable/containers.html#container-objects
+
+Binds to:
+  docker_daemon:
+    - `DockerDaemon`_
+
+Implements:
+  - :any:`PowerProtocol`
+
+.. code-block:: yaml
+
+   DockerDriver:
+     image_uri: "rastasheep/ubuntu-sshd:16.04"
+     container_name: "ubuntu-lg-example"
+     host_config: {"network_mode":"bridge"}
+     network_services: [{"port":22,"username":"root","password":"root"}]
+
+Arguments:
+  - image_uri (str): identifier of the docker image to use (may have a tag suffix)
+  - command (str): command to run in the container (optional, depends on image)
+  - volumes (list): list to configure volumes mounted inside the container (optional)
+  - container_name (str): name of the container
+  - environment (list): list of environment variables (optional)
+  - host_config (dict): dictionary of host configurations
+  - network_services (list): dictionaries that describe individual `NetworkService`_
+    instances that come alive when the container is created. The "address" argument
+    which `NetworkService`_ also requires will be derived automatically upon container
+    creation.
+
 Strategies
 ----------
 
@@ -1559,6 +1626,33 @@ to transition to the shell state:
 this command would transition from the boot loader into a Linux shell and
 activate the shelldriver.
 
+DockerShellStrategy
+~~~~~~~~~~~~~~~~~~~
+A DockerShellStrategy has three states:
+
+- unknown
+- off
+- shell
+
+
+To transition to the shell state:
+
+::
+
+   t = get_target("main")
+   s = DockerShellStrategy(t)
+   s.transition("shell")
+
+
+These commands would activate the docker driver which creates and starts
+a docker container. This will subsequently make `NetworkService`_ instance(s)
+available which can be used for e.g. ssh access.
+
+Note: Transitioning to the "off" state will make any `NetworkService`_
+instance(s) unresponsive - which may in turn invalidate ssh connection
+sharing. Therefore, during automated test suites, refrain from transitioning
+to the "off" state.
+
 Reporters
 ---------
 

docker-requirements.txt

@@ -0,0 +1 @@
+docker==3.7.2

examples/docker/README.md

@@ -0,0 +1,14 @@
+# Prerequisites #
+To run the docker example one has to have docker-ce installed and
+accessible via "unix:///var/run/docker.sock" (the default). The
+default docker bridge network also needs to be accessible from the
+pytest executor since the test tries to establish an ssh connection to
+the container (again the default after a standard installation of
+docker-ce).
+
+After following steps similar to [Getting started](https://labgrid.readthedocs.io/en/latest/getting_started.html#running-your-first-test) the demo can be run with:
+
+    pytest -s --lg-env env.yaml test_shell.py
+
+Successfully tested against Docker version 18.06.1-ce, build e68fc7a.
+But it should work with later versions as well.

examples/docker/conftest.py

@@ -0,0 +1,9 @@
+import pytest
+
+@pytest.fixture(scope='session')
+def command(target):
+    strategy = target.get_driver('DockerStrategy')
+    strategy.transition("shell")
+    shell = target.get_driver('CommandProtocol')
+    return shell
+

examples/docker/env.yaml

@@ -0,0 +1,14 @@
+targets:
+  main:
+    resources:
+      - DockerDaemon:
+          docker_daemon_url: "unix:///var/run/docker.sock"
+    drivers:
+      - DockerDriver:
+          image_uri: "rastasheep/ubuntu-sshd:16.04"
+          container_name: "ubuntu-lg-example"
+          host_config: {"network_mode":"bridge"}
+          network_services: [{"port":22,"username":"root","password":"root"}]
+      - DockerStrategy: {}
+      - SSHDriver:
+          keyfile: ""

examples/docker/test_shell.py

@@ -0,0 +1,11 @@
+def test_shell(command):
+    stdout, stderr, returncode = command.run('cat /proc/version')
+    assert returncode == 0
+    assert len(stdout) > 0
+    assert len(stderr) == 0
+    assert 'Linux' in stdout[0]
+
+    stdout, stderr, returncode = command.run('false')
+    assert returncode != 0
+    assert len(stdout) == 0
+    assert len(stderr) == 0

labgrid/driver/__init__.py

@@ -27,3 +27,4 @@
 from .filedigitaloutput import FileDigitalOutputDriver
 from .serialdigitaloutput import SerialPortDigitalOutputDriver
 from .xenadriver import XenaDriver
+from .dockerdriver import DockerDriver

labgrid/driver/dockerdriver.py

@@ -0,0 +1,110 @@
+"""
+Class for connecting to a docker daemon running on the host machine.
+"""
+
+import attr
+import logging
+
+from labgrid.factory import target_factory
+from labgrid.driver.common import Driver
+from labgrid.resource.docker import DockerDaemon, DockerConstants
+from labgrid.protocol.powerprotocol import PowerProtocol
+
+
+@target_factory.reg_driver
+@attr.s(cmp=False)
+class DockerDriver(PowerProtocol, Driver):
+    """The DockerDriver is used to create docker containers.
+    This is done via communication with a docker daemon.
+
+    When a container is created the container is labeled with an
+    cleanup strategy identifier. Currently only one strategy is
+    implemented.  This strategy simply deletes all labgrid created
+    containers before each test run. This is to ensure cleanup of
+    dangling containers from crashed tests or hanging containers.
+
+    Image pruning is not done by the driver.
+
+    For detailed information about the arguments see the
+    "Docker SDK for Python" documentation
+    https://docker-py.readthedocs.io/en/stable/containers.html#container-objects
+
+    Args:
+        bindings (dict): The labgrid bindings
+    Args passed to docker.create_container:
+        image_uri (str): The uri of the image to fetch
+        command (str): The command to execute once container has been created
+        volumes (list): The volumes to declare
+        environment (list): Docker environment variables to set
+        host_config (dict): Docker host configuration parameters
+        network_services (list): Sequence of dicts each specifying a network \
+                                 service that the docker container exposes.
+
+    """
+    bindings = {"docker_daemon": {DockerDaemon}}
+    image_uri = attr.ib(default=None, validator=attr.validators.optional(
+        attr.validators.instance_of(str)))
+    command = attr.ib(default=None, validator=attr.validators.optional(
+        attr.validators.instance_of(str)))
+    volumes = attr.ib(default=None, validator=attr.validators.optional(
+        attr.validators.instance_of(list)))
+    container_name = attr.ib(default=None, validator=attr.validators.optional(
+        attr.validators.instance_of(str)))
+    environment = attr.ib(
+        default=None, validator=attr.validators.optional(
+            attr.validators.instance_of(list)))
+    host_config = attr.ib(
+        default=None, validator=attr.validators.optional(
+            attr.validators.instance_of(dict)))
+    network_services = attr.ib(
+        default=None, validator=attr.validators.optional(
+            attr.validators.instance_of(list)))
+
+    def __attrs_post_init__(self):
+        self.logger = logging.getLogger("{}({})".format(self, self.target))
+        super().__attrs_post_init__()
+        self._client = None
+        self._container = None
+
+    def on_activate(self):
+        """ On activation:
+        1. Import docker module (_client and _container remain available)
+        2. Connect to the docker daemon
+        3. Pull requested image from docker registry if needed
+        4. Create the new container according to parameters from conf
+        """
+        import docker
+        self._client = docker.DockerClient(
+            base_url=self.docker_daemon.docker_daemon_url)
+        self._client.images.pull(self.image_uri)
+        self._container = self._client.api.create_container(
+            self.image_uri,
+            command=self.command,
+            volumes=self.volumes,
+            name=self.container_name,
+            environment=self.environment,
+            labels={
+                DockerConstants.DOCKER_LG_CLEANUP_LABEL:
+                DockerConstants.DOCKER_LG_CLEANUP_TYPE_AUTO},
+            host_config=self._client.api.create_host_config(
+                **self.host_config))
+
+    def on_deactivate(self):
+        """ Remove container after use"""
+        self._client.api.remove_container(self._container.get('Id'),
+                                          force=True)
+        self._client = None
+        self._container = None
+
+    def on(self):
+        """ Start the container created during activation """
+        self._client.api.start(container=self._container.get('Id'))
+
+    def off(self):
+        """ Stop the container created during activation """
+        self._client.api.stop(container=self._container.get('Id'))
+
+    def cycle(self):
+        """Cycle the docker container by stopping and starting it"""
+        self.off()
+        self.on()

labgrid/resource/__init__.py

@@ -13,3 +13,4 @@
 from .ykushpowerport import YKUSHPowerPort
 from .xenamanager import XenaManager
 from .flashrom import Flashrom, NetworkFlashrom
+from .docker import DockerManager, DockerDaemon, DockerConstants