Merge pull request #919 from andrzej-jankowski/3par

Scan postprocess - mount_shares plugin

src/ralph/scan/automerger.py

@@ -99,7 +99,7 @@ def _find_data_by_plugin(data, plugin):
             return results
 
 
-def _select_data(data, external_priorities={}, is_management=False):
+def select_data(data, external_priorities={}, is_management=False):
     selected_data = {}
     for component, result in data.iteritems():
         available_plugins = set()
@@ -187,7 +187,7 @@ def _save_job_results(job_id, start_ts):
             only_multiple=True,
         )
         append_merged_proposition(data, device, external_priorities)
-        selected_data = _select_data(data, external_priorities, is_management)
+        selected_data = select_data(data, external_priorities, is_management)
         set_device_data(device, selected_data, save_priority=SAVE_PRIORITY)
         device.save(priority=SAVE_PRIORITY)
     # now... we create new devices from `garbage`
@@ -215,7 +215,7 @@ def _save_job_results(job_id, start_ts):
                 garbage[plugin_name] = plugin_result
         if garbage:
             data = merge_data(garbage)
-            selected_data = _select_data(data, external_priorities)
+            selected_data = select_data(data, external_priorities)
             if all((
                 any(
                     (

src/ralph/scan/manual.py

@@ -354,9 +354,7 @@ def scan_address_job(
             # Run only when automerge mode is enabled and some change was
             # detected. When `change` state is not available just run it...
             save_job_results(job.id)
-        elif not called_from_ui and job.args and job.meta.get('changed', True):
-            # Run only when some change was detected. When `change` state is
-            # not available just run it...
+        elif not called_from_ui and job.args:
             try:
                 ip_obj = IPAddress.objects.select_related().get(
                     address=job.args[0]  # job.args[0] == ip_address
@@ -372,5 +370,5 @@ def scan_address_job(
                     except ImportError as e:
                         logger.error(unicode(e))
                     else:
-                        module.run_job(ip_obj)
+                        module.run_job(ip_obj, plugins_results=results)
     return results

src/ralph/scan/postprocess/cache_price.py

@@ -8,7 +8,7 @@
 from ralph.util import pricing
 
 
-def run_job(ip):
+def run_job(ip, **kwargs):
     device = ip.device
     if not device:
         return  # no device...

src/ralph/scan/postprocess/mount_shares.py

@@ -0,0 +1,225 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import logging
+
+from django.db.models import Q
+
+from ralph.discovery.models import (
+    Device,
+    DiskShare,
+    DiskShareMount,
+    IPAddress,
+)
+from ralph.scan.automerger import select_data
+from ralph.scan.data import (
+    append_merged_proposition,
+    find_devices,
+    get_external_results_priorities,
+    get_device_data,
+    merge_data,
+)
+
+
+logger = logging.getLogger("MOUNT_SHARES")
+
+
+def _shares_in_results(data):
+    shares_in_device, shares_in_subdevice = False, False
+    for plugin_name, plugin_result in data.iteritems():
+        if plugin_result['status'] == 'error':
+            continue
+        if 'device' not in plugin_result:
+            continue
+        if 'disk_shares' in plugin_result['device']:
+            shares_in_device = True
+        for subdevice in plugin_result['device'].get('subdevices', []):
+            if 'disk_shares' in subdevice:
+                shares_in_subdevice = True
+                break
+    return shares_in_device, shares_in_subdevice
+
+
+def _it_is_3par(wwn):
+    return wwn[1:7] == '0002AC'
+
+
+def _create_or_update_share_mount(ip, device, share_mount):
+    share_address = None
+    if share_mount.get('address'):
+        try:
+            share_address = IPAddress.objects.get(
+                address=share_mount['address']
+            )
+        except IPAddress.DoesNotExist:
+            pass
+    try:
+        share = DiskShare.objects.get(
+            wwn=share_mount['serial_number']
+        )
+    except DiskShare.DoesNotExist:
+        logger.warning(
+            '%s%sNo share found for share mount: '
+            'WWN=%s; Volume=%s; device=%s; venture=%s; device_ip=%s' % (
+                '(3PAR) ' if _it_is_3par(
+                    share_mount['serial_number']
+                ) else '',
+                '(virtual) ' if share_mount.get(
+                    'is_virtual', False
+                ) else '',
+                share_mount['serial_number'],
+                share_mount.get('volume'),
+                device,
+                device.venture,
+                ip.address
+            )
+        )
+        return
+    try:
+        mount = DiskShareMount.objects.get(device=device, share=share)
+    except DiskShareMount.DoesNotExist:
+        mount = DiskShareMount(
+            device=device,
+            share=share,
+        )
+        logger.info(
+            '%s%sCreate mount point on device %s (venture=%s). '
+            'Share WWN=%s; Volume=%s; Share label=%s; IP address=%s' % (
+                '(3PAR) ' if _it_is_3par(
+                    share_mount['serial_number']
+                ) else '',
+                '(virtual) ' if share_mount.get(
+                    'is_virtual', False
+                ) else '',
+                device,
+                device.venture,
+                share_mount['serial_number'],
+                share_mount.get('volume'),
+                share.label,
+                ip.address
+            )
+        )
+    else:
+        logger.info(
+            '%s%sMount point on device %s (venture=%s) exists. '
+            'Share WWN=%s; Volume=%s; Share label=%s; IP address=%s' % (
+                '(3PAR) ' if _it_is_3par(
+                    share_mount['serial_number']
+                ) else '',
+                '(virtual) ' if share_mount.get(
+                    'is_virtual', False
+                ) else '',
+                device,
+                device.venture,
+                share_mount['serial_number'],
+                share_mount.get('volume'),
+                share.label,
+                ip.address
+            )
+        )
+    if share_mount.get('size'):
+        mount.size = share_mount['size']
+    if share_mount.get('volume'):
+        mount.volume = share_mount['volume']
+    if share_address:
+        mount.address = share_address
+    mount.is_virtual = share_mount.get('is_virtual', False)
+    # mount.save()  # temporary
+
+
+def _append_shares_to_device(ip, device, data, external_priorities={}):
+    device_data = get_device_data(device)
+    full_data = merge_data(
+        data,
+        {
+            'database': {'device': device_data},
+        },
+        only_multiple=True,
+    )
+    append_merged_proposition(full_data, device, external_priorities)
+    selected_data = select_data(full_data, external_priorities)
+    for share_mount in selected_data.get('disk_shares', []):
+        _create_or_update_share_mount(ip, device, share_mount)
+
+
+def _append_shares_to_subdevice(ip, device, shares_data):
+    for share_mount in shares_data:
+        _create_or_update_share_mount(ip, device, share_mount)
+
+
+def _mount_shares(ip, data):
+    shares_in_device, shares_in_subdevice = _shares_in_results(data)
+    if not shares_in_device and not shares_in_subdevice:
+        return  # No reason to run it.
+    external_priorities = get_external_results_priorities(data)
+    # main devices
+    if shares_in_device:
+        devices = find_devices(data)
+        if len(devices) == 1:
+            _append_shares_to_device(ip, devices[0], data, external_priorities)
+        else:
+            if len(devices) == 0:
+                logger.warning(
+                    'No device found for the IP address %s. '
+                    'There are some shares.' % ip.address
+                )
+            else:
+                logger.warning(
+                    'Many devices found for the IP address %s. '
+                    'There are some shares.' % ip.address
+                )
+    # subdevices
+    if shares_in_subdevice:
+        for plugin_name, plugin_result in data.iteritems():
+            if plugin_result['status'] == 'error':
+                continue
+            if 'device' not in plugin_result:
+                continue
+            for subdev_data in plugin_result['device'].get('subdevices', []):
+                if 'disk_shares' not in subdev_data:
+                    continue
+                serials = set()
+                if subdev_data.get('serial_number'):
+                    serials.add(subdev_data['serial_number'])
+                macs = set()
+                for mac in subdev_data.get('mac_addresses', []):
+                    macs.add(mac)
+                devices = Device.admin_objects.filter(
+                    Q(sn__in=serials) |
+                    Q(ethernet__mac__in=macs)
+                ).distinct()
+                if len(devices) == 1:
+                    _append_shares_to_subdevice(
+                        ip,
+                        devices[0],
+                        subdev_data['disk_shares']
+                    )
+                else:
+                    if len(devices) == 0:
+                        logger.warning(
+                            'No subdevice found for the hypervisor IP '
+                            'address %s. SNs: %r; MACs: %r. There are some '
+                            'shares.' % (
+                                ip.address,
+                                serials,
+                                macs
+                            )
+                        )
+                    else:
+                        logger.warning(
+                            'Many subdevices found for the hypervisor IP '
+                            'address %s. Devices: %r. There are some '
+                            'shares.' % (
+                                ip.address,
+                                devices
+                            )
+                        )
+
+
+def run_job(ip, **kwargs):
+    data = kwargs.get('plugins_results', {})
+    _mount_shares(ip, data)

src/ralph/scan/postprocess/position.py

@@ -64,7 +64,7 @@ def _connect_dc(ip_address, device):
         device.save()
 
 
-def run_job(ip):
+def run_job(ip, **kwargs):
     device = ip.device
     if not device:
         return  # no device...

src/ralph/settings.py

@@ -366,6 +366,7 @@
 SCAN_POSTPROCESS_ENABLED_JOBS = [
     'ralph.scan.postprocess.position',
     'ralph.scan.postprocess.cache_price',
+    'ralph.scan.postprocess.mount_shares',
 ]
 
 #