Subscriber for virtual server (#2513)

Add subscriber for custom fields

src/ralph/ralph2_sync/subscribers.py

@@ -4,6 +4,7 @@
 from functools import wraps
 
 import pyhermes
+from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
 from django.db import transaction
 
@@ -24,6 +25,7 @@
 from ralph.networks.models import IPAddress
 from ralph.ralph2_sync.helpers import WithSignalDisabled
 from ralph.ralph2_sync.publishers import sync_dc_asset_to_ralph2
+from ralph.virtual.models import VirtualServer, VirtualServerType
 
 logger = logging.getLogger(__name__)
 
@@ -45,6 +47,39 @@ def _get_publisher_signal_info(func):
     }
 
 
+def _get_service_env(data, service_key='service', env_key='environment'):
+    """
+    Return service-env instance based on data dict.
+    """
+    if service_key not in data and env_key not in data:
+        return None
+    service = data[service_key]
+    environment = data[env_key]
+    if not service or not environment:
+        return None
+    service_env = ServiceEnvironment.objects.get(
+        service__uid=service,
+        environment=ImportedObjects.get_object_from_old_pk(
+            Environment, environment
+        )
+    )
+    return service_env
+
+
+def _get_configuration_path_from_venture_role(venture_role_id):
+    if venture_role_id is None:
+        return
+    try:
+        return ImportedObjects.get_object_from_old_pk(
+            ConfigurationClass, venture_role_id
+        )
+    except ImportedObjectDoesNotExist:
+        logger.error('VentureRole {} not found when syncing'.format(
+            venture_role_id
+        ))
+    return None
+
+
 class sync_subscriber(pyhermes.subscriber):
     """
     Log additional exception when sync has failed.
@@ -65,8 +100,10 @@ def exception_wrapper(*args, **kwargs):
                     ))
                 try:
                     return func(*args, **kwargs)
-                except:
-                    logger.exception('Exception during syncing')
+                except Exception as e:
+                    logger.exception(
+                        'Exception during syncing {}'.format(str(e))
+                    )
         return exception_wrapper
 
 
@@ -130,30 +167,16 @@ def sync_device_to_ralph3(data):
         else:
             del dca.management_ip
     if 'service' in data and 'environment' in data:
-        service = data['service']
-        environment = data['environment']
-        dca.service_env = ServiceEnvironment.objects.get(
-            service__uid=service,
-            environment=ImportedObjects.get_object_from_old_pk(
-                Environment, environment
-            )
-        )
+        dca.service_env = _get_service_env(data)
     if 'venture_role' in data:
         if data['venture_role']:
-            try:
-                dca.configuration_path = ImportedObjects.get_object_from_old_pk(
-                    ConfigurationClass, data['venture_role']
-                )
-            except ImportedObjectDoesNotExist:
-                logger.error('VentureRole {} not found when syncing {}'.format(
-                    data['venture_role'], data['id']
-                ))
-        else:
-            dca.configuration_path = None
+            dca.configuration_path = _get_configuration_path_from_venture_role(
+                venture_role_id=data['venture_role']
+            )
+    dca.save()
     if 'custom_fields' in data:
         for field, value in data['custom_fields'].items():
             dca.update_custom_field(field, value)
-    dca.save()
 
 
 @sync_subscriber(
@@ -267,3 +290,55 @@ def sync_venture_role_to_ralph3(data):
     if creating:
         ImportedObjects.create(conf_class, data['id'])
     logger.info('Synced configuration class {}'.format(conf_class))
+
+
+def _get_obj(model_class, obj_id, creating=False):
+    """
+    Custom get or create based on imported objects.
+    """
+    try:
+        obj = ImportedObjects.get_object_from_old_pk(
+            model_class, obj_id
+        )
+        return obj, False
+    except ImportedObjectDoesNotExist:
+        obj = None
+        if creating:
+            obj = model_class()
+        logger.info(
+            '{} class ({}) not found'.format(
+                model_class, obj_id
+            )
+        )
+        return obj, True
+
+
+@sync_subscriber(topic='sync_virtual_server_to_ralph3')
+def sync_virtual_server_to_ralph3(data):
+    virtual_type = settings.RALPH2_RALPH3_VIRTUAL_SERVER_TYPE_MAPPING.get(data['type'])  # noqa
+    if virtual_type is None:
+        logger.info(
+            'Type {} not found in mapping dict'.format(
+                data['type']
+            )
+        )
+        virtual_type = data['type']
+    virtual_server, created = _get_obj(VirtualServer, data['id'], creating=True)
+    service_env = _get_service_env(data)
+    virtual_server.sn = data['sn']
+    virtual_server.hostname = data['hostname']
+    virtual_server.service_env = service_env
+    virtual_server.configuration_path = _get_configuration_path_from_venture_role(  # noqa
+        venture_role_id=data['venture_role']
+    )
+    virtual_server.type = VirtualServerType.objects.get_or_create(
+        name=virtual_type
+    )[0]
+    hypervisor, _ = _get_obj(DataCenterAsset, data['parent_id'])
+    virtual_server.parent = hypervisor
+    virtual_server.save()
+    if 'custom_fields' in data:
+        for field, value in data['custom_fields'].items():
+            virtual_server.update_custom_field(field, value)
+    if created:
+        ImportedObjects.create(virtual_server, data['id'])

src/ralph/ralph2_sync/tests/test_subscribers.py

@@ -1,5 +1,6 @@
 from django.contrib.contenttypes.models import ContentType
 from django.test import TestCase
+from django.test.utils import override_settings
 
 from ralph.accounts.tests.factories import TeamFactory
 from ralph.assets.models import (
@@ -9,7 +10,10 @@
 )
 from ralph.assets.tests.factories import (
     ConfigurationClassFactory,
-    ConfigurationModuleFactory
+    ConfigurationModuleFactory,
+    EnvironmentFactory,
+    ServiceEnvironmentFactory,
+    ServiceFactory
 )
 from ralph.data_center.tests.factories import DataCenterAssetFactory
 from ralph.data_importer.models import (
@@ -23,8 +27,19 @@
     sync_custom_fields_to_ralph3,
     sync_device_to_ralph3,
     sync_venture_role_to_ralph3,
-    sync_venture_to_ralph3
+    sync_venture_to_ralph3,
+    sync_virtual_server_to_ralph3
 )
+from ralph.virtual.models import VirtualServer, VirtualServerType
+from ralph.virtual.tests.factories import VirtualServerFactory
+
+
+def _create_imported_object(factory, old_id, factory_kwargs=None):
+    if factory_kwargs is None:
+        factory_kwargs = {}
+    obj = factory(**factory_kwargs)
+    ImportedObjects.create(obj, old_id)
+    return obj
 
 
 class Ralph2SyncACKTestCase(TestCase):
@@ -242,3 +257,125 @@ def test_venture_role_with_non_existing_parent(self):
             ImportedObjects.get_object_from_old_pk(
                 ConfigurationModule, 33
             )
+
+
+class Ralph2SyncVirtualServerTestCase(TestCase):
+    def setUp(self):
+        self.data = {
+            'id': 1,
+            'type': 'unknown',
+            'hostname': None,
+            'sn': None,
+            'service': None,
+            'environment': None,
+            'venture_role': None,
+            'parent_id': None,
+            'custom_fields': {}
+        }
+
+    def sync(self):
+        obj = self._create_imported_virtual_server()
+        sync_virtual_server_to_ralph3(self.data)
+        obj.refresh_from_db()
+        return obj
+
+    def _create_imported_virtual_server(self, old_id=None):
+        return _create_imported_object(
+            factory=VirtualServerFactory,
+            old_id=old_id if old_id else self.data['id']
+        )
+
+    def test_new_virtual_server_should_create_imported_object(self):
+        self.assertEqual(VirtualServer.objects.count(), 0)
+        sync_virtual_server_to_ralph3(self.data)
+        self.assertEqual(VirtualServer.objects.count(), 1)
+        vs = ImportedObjects.get_object_from_old_pk(
+            VirtualServer, self.data['id']
+        )
+        self.assertEqual(vs.hostname, self.data['hostname'])
+
+    def test_existing_virtual_server_should_updated(self):
+        vs = self._create_imported_virtual_server()
+        sync_virtual_server_to_ralph3(self.data)
+        self.assertEqual(VirtualServer.objects.count(), 1)
+        vs = ImportedObjects.get_object_from_old_pk(
+            VirtualServer, self.data['id']
+        )
+        self.assertEqual(vs.hostname, self.data['hostname'])
+
+    def test_sync_should_change_hostname(self):
+        vs = self._create_imported_virtual_server()
+        self.data['hostname'] = 'new.hostname.dc.net'
+        self.assertNotEqual(self.data['hostname'], vs.hostname)
+        sync_virtual_server_to_ralph3(self.data)
+        vs.refresh_from_db()
+        self.assertEqual(vs.hostname, self.data['hostname'])
+
+    def test_sync_should_change_sn(self):
+        vs = self._create_imported_virtual_server()
+        self.data['sn'] = 'sn-123'
+        self.assertNotEqual(self.data['sn'], vs.sn)
+        sync_virtual_server_to_ralph3(self.data)
+        vs.refresh_from_db()
+        self.assertEqual(vs.sn, self.data['sn'])
+
+    def test_sync_should_create_type_if_not_exist(self):
+        self.data['type'] = 'new unique type'
+        type_exists = VirtualServerType.objects.filter(name=self.data['type']).exists  # noqa
+        self.assertFalse(type_exists())
+        sync_virtual_server_to_ralph3(self.data)
+        self.assertTrue(type_exists())
+
+    @override_settings(
+        RALPH2_RALPH3_VIRTUAL_SERVER_TYPE_MAPPING={
+            'type in R2': 'type in R3'
+        },
+    )
+    def test_sync_should_create_type_and_mapping(self):
+        self.data['type'] = 'type in R2'
+        type_exists = VirtualServerType.objects.filter(name='type in R3').exists  # noqa
+        self.assertFalse(type_exists())
+        sync_virtual_server_to_ralph3(self.data)
+        self.assertTrue(type_exists())
+
+    def test_sync_should_change_service_env(self):
+        self.data['service'] = '123'
+        service = _create_imported_object(
+            ServiceFactory, self.data['service'], factory_kwargs={
+                'uid': self.data['service']
+            }
+        )
+        self.data['environment'] = '124'
+        env = _create_imported_object(
+            EnvironmentFactory, self.data['environment']
+        )
+        se = ServiceEnvironmentFactory(service=service, environment=env)
+        vs = self.sync()
+        self.assertEqual(vs.service_env, se)
+
+    def test_sync_should_change_configuration_path(self):
+        self.data['venture_role'] = 'venture_role_123'
+        conf_class = _create_imported_object(
+            ConfigurationClassFactory, self.data['venture_role']
+        )
+        vs = self.sync()
+        self.assertEqual(vs.configuration_path, conf_class)
+
+    def test_sync_should_change_parent(self):
+        self.data['parent_id'] = '123333'
+        parent = _create_imported_object(
+            DataCenterAssetFactory, self.data['parent_id']
+        )
+        vs = self.sync()
+        self.assertEqual(vs.parent.id, parent.id)
+
+    def test_sync_should_change_custom_fields(self):
+        cf = CustomField.objects.create(
+            name='test_str', type=CustomFieldTypes.STRING,
+        )
+        custom_fields = {
+            cf.name: 'test'
+        }
+        self.data['custom_fields'] = custom_fields
+        vs = self.sync()
+        self.assertEqual(vs.custom_fields_as_dict, custom_fields)

src/ralph/settings/base.py

@@ -360,3 +360,8 @@ def os_env_true(var, default=''):
 RALPH2_HERMES_SYNC_FUNCTIONS = json.loads(
     os.environ.get('RALPH2_HERMES_SYNC_FUNCTIONS', '[]')
 )
+
+# mapping model's name to type
+RALPH2_RALPH3_VIRTUAL_SERVER_TYPE_MAPPING = json.loads(
+    os.environ.get('RALPH2_RALPH3_VIRTUAL_SERVER_TYPE_MAPPING', '{}')
+)

src/ralph/virtual/migrations/0007_auto_20160630_0949.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import ralph.lib.mixins.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('virtual', '0006_virtualcomponent_model_name'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='virtualserver',
+            name='sn',
+            field=ralph.lib.mixins.fields.NullableCharField(max_length=200, null=True, blank=True, verbose_name='SN', default=None, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='virtualservertype',
+            name='name',
+            field=models.CharField(verbose_name='name', max_length=255, unique=True),
+        ),
+    ]

src/ralph/virtual/models.py

@@ -215,7 +215,7 @@ class VirtualComponent(Component):
 
 
 class VirtualServerType(
-    NamedMixin.NonUnique,
+    NamedMixin,
     TimeStampMixin,
     models.Model
 ):
@@ -247,9 +247,12 @@ class VirtualServer(AdminAbsoluteUrlMixin, NetworkableBaseObject, BaseObject):
         verbose_name=_('hostname'),
         unique=True,
     )
-    sn = models.CharField(
+    sn = NullableCharField(
         max_length=200,
         verbose_name=_('SN'),
+        blank=True,
+        default=None,
+        null=True,
         unique=True,
     )
     # TODO: remove this field