Merge pull request #31 from openwisp/connections

openwisp_controller.connection module (SSH connections)

Pipfile

@@ -11,6 +11,8 @@ coverage = "*"
 coveralls = "*"
 isort = "*"
 flake8 = "*"
+mock = "*"
+mock-ssh-server = {version = ">=0.5.0,<0.6.0"}
 
 [scripts]
 lint = "python -m flake8"

README.rst

@@ -188,15 +188,75 @@ Add the following settings to ``settings.py``:
 
     urlpatterns += staticfiles_urlpatterns()
 
+Settings
+--------
+
+``OPENWISP_CONNECTORS``
+~~~~~~~~~~~~~~~~~~~~~~~
+
++--------------+--------------------------------------------------------------------+
+| **type**:    | ``tuple``                                                          |
++--------------+--------------------------------------------------------------------+
+| **default**: | .. code-block:: python                                             |
+|              |                                                                    |
+|              |   (                                                                |
+|              |     ('openwisp_controller.connection.connectors.ssh.Ssh', 'SSH'),  |
+|              |   )                                                                |
++--------------+--------------------------------------------------------------------+
+
+Available connector classes. Connectors are python classes that specify ways
+in which OpenWISP can connect to devices in order to launch commands.
+
+``OPENWISP_UPDATE_STRATEGIES``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
++--------------+----------------------------------------------------------------------------------------+
+| **type**:    | ``tuple``                                                                              |
++--------------+----------------------------------------------------------------------------------------+
+| **default**: | .. code-block:: python                                                                 |
+|              |                                                                                        |
+|              |   (                                                                                    |
+|              |     ('openwisp_controller.connection.connectors.openwrt.ssh.OpenWrt', 'OpenWRT SSH'),  |
+|              |   )                                                                                    |
++--------------+----------------------------------------------------------------------------------------+
+
+Available update strategies. An update strategy is a subclass of a
+connector class which defines an ``update_config`` method which is
+in charge of updating the configuratio of the device.
+
+This operation is launched in a background worker when the configuration
+of a device is changed.
+
+It's possible to write custom update strategies and add them to this
+setting to make them available in OpenWISP.
+
+``OPENWISP_CONFIG_UPDATE_MAPPING``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
++--------------+--------------------------------------------------------------------+
+| **type**:    | ``dict``                                                           |
++--------------+--------------------------------------------------------------------+
+| **default**: | .. code-block:: python                                             |
+|              |                                                                    |
+|              |   {                                                                |
+|              |     'netjsonconfig.OpenWrt': OPENWISP_UPDATE_STRATEGIES[0][0],     |
+|              |   }                                                                |
++--------------+--------------------------------------------------------------------+
+
+A dictionary that maps configuration backends to update strategies in order to
+automatically determine the update strategy of a device connection if the
+update strategy field is left blank by the user.
+
 Installing for development
 --------------------------
 
-Install sqlite:
+Install the dependencies:
 
 .. code-block:: shell
 
-    sudo apt-get install sqlite3 libsqlite3-dev libsqlite3-mod-spatialite openssl libssl-dev
-    sudo apt-get install gdal-bin libproj-dev libgeos-dev libspatialite-dev
+    sudo apt-get install sqlite3 libsqlite3-dev openssl libssl-dev
+    sudo apt-get install gdal-bin libproj-dev libgeos-dev libspatialite-dev libsqlite3-mod-spatialite
+    sudo apt-get install redis
 
 Install your forked repo with `pipenv <https://pipenv.readthedocs.io/en/latest/>`_:
 
@@ -215,6 +275,12 @@ Create database:
     pipenv run ./manage.py migrate
     pipenv run ./manage.py createsuperuser
 
+Launch celery worker (for background jobs):
+
+.. code-block:: shell
+
+    celery -A openwisp2 worker -l info
+
 Launch development server:
 
 .. code-block:: shell

openwisp_controller/config/tests/test_admin.py

@@ -41,6 +41,15 @@ class TestAdmin(CreateConfigTemplateMixin, TestAdminMixin,
         'config-INITIAL_FORMS': 0,
         'config-MIN_NUM_FORMS': 0,
         'config-MAX_NUM_FORMS': 1,
+        # openwisp_controller.connection
+        'deviceconnection_set-TOTAL_FORMS': 0,
+        'deviceconnection_set-INITIAL_FORMS': 0,
+        'deviceconnection_set-MIN_NUM_FORMS': 0,
+        'deviceconnection_set-MAX_NUM_FORMS': 1000,
+        'deviceip_set-TOTAL_FORMS': 0,
+        'deviceip_set-INITIAL_FORMS': 0,
+        'deviceip_set-MIN_NUM_FORMS': 0,
+        'deviceip_set-MAX_NUM_FORMS': 1000,
     }
     # WARNING - WATCHOUT
     # this class attribute is changed dinamically

openwisp_controller/config/tests/test_controller.py

@@ -126,7 +126,7 @@ def test_report_status_404_disabled_org(self):
         org = self._create_org(is_active=False)
         c = self._create_config(organization=org)
         response = self.client.post(reverse('controller:report_status', args=[c.device.pk]),
-                                    {'key': c.device.key, 'status': 'running'})
+                                    {'key': c.device.key, 'status': 'applied'})
         self.assertEqual(response.status_code, 404)
 
     def test_checksum_200(self):

openwisp_controller/config/views.py

@@ -13,7 +13,7 @@ def get_default_templates(request, organization_id):
     """
     user = request.user
     authenticated = user.is_authenticated
-    if callable(authenticated):
+    if callable(authenticated):  # pragma: nocover
         authenticated = authenticated()
     if not authenticated and not user.is_staff:
         return HttpResponse(status=403)

openwisp_controller/connection/__init__.py

@@ -0,0 +1 @@
+default_app_config = 'openwisp_controller.connection.apps.ConnectionConfig'

openwisp_controller/connection/admin.py

@@ -0,0 +1,49 @@
+from django.contrib import admin
+
+from openwisp_users.multitenancy import MultitenantOrgFilter
+from openwisp_utils.admin import TimeReadonlyAdminMixin
+
+from ..admin import MultitenantAdminMixin
+from ..config.admin import DeviceAdmin
+from .models import Credentials, DeviceConnection, DeviceIp
+
+
+@admin.register(Credentials)
+class CredentialsAdmin(MultitenantAdminMixin, TimeReadonlyAdminMixin, admin.ModelAdmin):
+    list_display = ('name',
+                    'organization',
+                    'connector',
+                    'auto_add',
+                    'created',
+                    'modified')
+    list_filter = [('organization', MultitenantOrgFilter),
+                   'connector']
+    list_select_related = ('organization',)
+
+
+class DeviceIpInline(admin.TabularInline):
+    model = DeviceIp
+    exclude = ('created', 'modified')
+    extra = 0
+
+    def get_queryset(self, request):
+        qs = super(DeviceIpInline, self).get_queryset(request)
+        return qs.order_by('priority')
+
+
+class DeviceConnectionInline(MultitenantAdminMixin, admin.StackedInline):
+    model = DeviceConnection
+    exclude = ['params', 'created', 'modified']
+    readonly_fields = ['is_working', 'failure_reason', 'last_attempt']
+    extra = 0
+
+    multitenant_shared_relations = ('credentials',)
+
+    def get_queryset(self, request):
+        """
+        Override MultitenantAdminMixin.get_queryset() because it breaks
+        """
+        return super(admin.StackedInline, self).get_queryset(request)
+
+
+DeviceAdmin.inlines += [DeviceConnectionInline, DeviceIpInline]

openwisp_controller/connection/apps.py

@@ -0,0 +1,52 @@
+from celery.task.control import inspect
+from django.apps import AppConfig
+from django.db.models.signals import post_save
+from django.utils.translation import ugettext_lazy as _
+from django_netjsonconfig.signals import config_modified
+
+_TASK_NAME = 'openwisp_controller.connection.tasks.update_config'
+
+
+class ConnectionConfig(AppConfig):
+    name = 'openwisp_controller.connection'
+    label = 'connection'
+    verbose_name = _('Network Device Credentials')
+
+    def ready(self):
+        """
+        connects the ``config_modified`` signal
+        to the ``update_config`` celery task
+        which will be executed in the background
+        """
+        config_modified.connect(self.config_modified_receiver,
+                                dispatch_uid='connection.update_config')
+
+        from ..config.models import Config
+        from .models import Credentials
+
+        post_save.connect(Credentials.auto_add_credentials_to_device,
+                          sender=Config,
+                          dispatch_uid='connection.auto_add_credentials')
+
+    @classmethod
+    def config_modified_receiver(cls, **kwargs):
+        from .tasks import update_config
+        d = kwargs['device']
+        conn_count = d.deviceconnection_set.count()
+        # if device has no connection specified
+        # or update is already in progress, stop here
+        if conn_count < 1 or cls._is_update_in_progress(d.id):
+            return
+        update_config.delay(d.id)
+
+    @classmethod
+    def _is_update_in_progress(cls, device_id):
+        active = inspect().active()
+        if not active:
+            return False
+        # check if there's any other running task before adding it
+        for task_list in active.values():
+            for task in task_list:
+                if task['name'] == _TASK_NAME and str(device_id) in task['args']:
+                    return True
+        return False

openwisp_controller/connection/connectors/openwrt/ssh.py

@@ -0,0 +1,6 @@
+from ..ssh import Ssh
+
+
+class OpenWrt(Ssh):
+    def update_config(self):
+        self.exec_command('/etc/init.d/openwisp_config restart')

openwisp_controller/connection/connectors/ssh.py

@@ -0,0 +1,133 @@
+import logging
+import socket
+import sys
+
+import paramiko
+from django.utils.functional import cached_property
+from jsonschema import validate
+from jsonschema.exceptions import ValidationError as SchemaError
+
+if sys.version_info.major > 2:  # pragma: nocover
+    from io import StringIO
+else:  # pragma: nocover
+    from StringIO import StringIO
+
+
+logger = logging.getLogger(__name__)
+SSH_CONNECTION_TIMEOUT = 5
+SSH_AUTH_TIMEOUT = 2
+SSH_COMMAND_TIMEOUT = 30
+
+
+class Ssh(object):
+    schema = {
+        "$schema": "http://json-schema.org/draft-04/schema#",
+        "type": "object",
+        "additionalProperties": False,
+        "required": ["username"],
+        "properties": {
+            "username": {"type": "string"},
+            "password": {"type": "string"},
+            "key": {"type": "string"},
+            "port": {"type": "integer"},
+        }
+    }
+
+    def __init__(self, params, addresses):
+        self._params = params
+        self.addresses = addresses
+        self.shell = paramiko.SSHClient()
+        self.shell.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+    @classmethod
+    def validate(cls, params):
+        validate(params, cls.schema)
+        cls.custom_validation(params)
+
+    @classmethod
+    def custom_validation(cls, params):
+        if 'password' not in params and 'key' not in params:
+            raise SchemaError('Missing password or key')
+
+    @cached_property
+    def params(self):
+        params = self._params.copy()
+        if 'key' in params:
+            key_fileobj = StringIO(params.pop('key'))
+            params['pkey'] = paramiko.RSAKey.from_private_key(key_fileobj)
+        return params
+
+    def connect(self):
+        success = False
+        exception = None
+        for address in self.addresses:
+            try:
+                self.shell.connect(address,
+                                   timeout=SSH_CONNECTION_TIMEOUT,
+                                   auth_timeout=SSH_AUTH_TIMEOUT,
+                                   **self.params)
+            except Exception as e:
+                exception = e
+            else:
+                success = True
+                break
+        if not success:
+            raise exception
+
+    def disconnect(self):
+        self.shell.close()
+
+    def exec_command(self, command, timeout=SSH_COMMAND_TIMEOUT,
+                     exit_codes=[0], raise_unexpected_exit=True):
+        """
+        Executes a command and performs the following operations
+        - logs executed command
+        - logs standard output
+        - logs standard error
+        - aborts on exceptions
+        - raises socket.timeout exceptions
+        """
+        print('$:> {0}'.format(command))
+        # execute commmand
+        try:
+            stdin, stdout, stderr = self.shell.exec_command(command,
+                                                            timeout=timeout)
+        # re-raise socket.timeout to avoid being catched
+        # by the subsequent `except Exception as e` block
+        except socket.timeout:
+            raise socket.timeout()
+        # any other exception will abort the operation
+        except Exception as e:
+            logger.exception(e)
+            raise e
+        # store command exit status
+        exit_status = stdout.channel.recv_exit_status()
+        # log standard output
+        output = stdout.read().decode('utf8').strip()
+        if output:
+            print(output)
+        # log standard error
+        error = stderr.read().decode('utf8').strip()
+        if error:
+            print(error)
+        # abort the operation if any of the command
+        # returned with a non-zero exit status
+        if exit_status not in exit_codes and raise_unexpected_exit:
+            print('# Previus command failed, aborting...')
+            message = error if error else output
+            raise Exception(message)
+        return output, exit_status
+
+    def update_config(self):  # pragma: no cover
+        raise NotImplementedError()
+
+    # TODO: this method is not used yet
+    # but will be necessary in the future to support other OSes
+    # def upload(self, fl, remote_path):
+    #     scp = SCPClient(self.shell.get_transport())
+    #     if not hasattr(fl, 'getvalue'):
+    #         fl_memory = BytesIO(fl.read())
+    #         fl.seek(0)
+    #         fl = fl_memory
+    #     scp.putfo(fl, remote_path)
+    #     scp.close()