network components in API improvements & performance improvements for…

… DataCenterAsset and Virtual Server (#2484)

Network components in API improvements

* filter ethernet by ipaddress
* attach IP info in ethernet
* filter IP by mac address
* attach ethernet (and simple IP) info for DataCenterAsset and VirtualServer

* various filters and performance (sql queries count) optimizations

src/ralph/assets/api/serializers.py

@@ -30,6 +30,7 @@
 from ralph.assets.models.components import Ethernet
 from ralph.lib.custom_fields.api import WithCustomFieldsSerializerMixin
 from ralph.licences.api_simple import SimpleBaseObjectLicenceSerializer
+from ralph.networks.api_simple import IPAddressSimpleSerializer
 
 
 class BusinessSegmentSerializer(RalphAPISerializer):
@@ -255,7 +256,16 @@ class Meta(BaseObjectSerializer.Meta):
         model = Asset
 
 
-class EthernetSerializer(RalphAPISerializer):
+class EthernetSimpleSerializer(RalphAPISerializer):
+    ipaddress = IPAddressSimpleSerializer()
+
+    class Meta:
+        model = Ethernet
+        fields = ('id', 'mac', 'ipaddress', 'url')
+
+
+class EthernetSerializer(EthernetSimpleSerializer):
+
     class Meta:
         model = Ethernet
         depth = 1

src/ralph/assets/api/views.py

@@ -6,6 +6,7 @@
 from ralph.api.utils import PolymorphicViewSetMixin
 from ralph.assets import models
 from ralph.assets.api import serializers
+from ralph.lib.custom_fields.models import CustomFieldValue
 from ralph.licences.api import BaseObjectLicenceViewSet
 from ralph.licences.models import BaseObjectLicence
 from ralph.networks.models import IPAddress
@@ -77,6 +78,10 @@ class BaseObjectViewSet(PolymorphicViewSetMixin, RalphAPIViewSet):
         Prefetch('licences', queryset=BaseObjectLicence.objects.select_related(
             *BaseObjectLicenceViewSet.select_related
         )),
+        Prefetch(
+            'custom_fields',
+            queryset=CustomFieldValue.objects.select_related('custom_field')
+        )
     ]
     filter_fields = [
         'id', 'service_env', 'service_env', 'service_env__service__uid',
@@ -100,7 +105,7 @@ class AssetHolderViewSet(RalphAPIViewSet):
 class EthernetViewSet(RalphAPIViewSet):
     queryset = models.Ethernet.objects.all()
     serializer_class = serializers.EthernetSerializer
-    filter_fields = ['base_object']
+    filter_fields = ['base_object', 'ipaddress__address']
     prefetch_related = ['model', 'base_object', 'base_object__tags']
 
     def destroy(self, request, *args, **kwargs):

src/ralph/assets/tests/factories.py

@@ -15,7 +15,7 @@
     ServiceEnvironment
 )
 from ralph.assets.models.base import BaseObject
-from ralph.assets.models.choices import ComponentType, ObjectModelType
+from ralph.assets.models.choices import ObjectModelType
 from ralph.assets.models.components import Ethernet
 from ralph.assets.models.configuration import (
     ConfigurationClass,
@@ -185,6 +185,13 @@ class Meta:
         django_get_or_create = ['label']
 
 
+class EthernetWithIPAddressFactory(EthernetFactory):
+    ipaddress = factory.RelatedFactory(
+        'ralph.networks.tests.factories.IPAddressWithNetworkFactory',
+        'ethernet'
+    )
+
+
 class ConfigurationModuleFactory(DjangoModelFactory):
     name = factory.Iterator(['ralph', 'allegro', 'auth', 'order'])
 

src/ralph/assets/tests/test_api.py

@@ -760,6 +760,28 @@ def setUp(self):
         super().setUp()
         self.ip = IPAddressFactory(dhcp_expose=True)
         self.eth = self.ip.ethernet
+        self.eth2 = EthernetFactory()
+
+    def test_get_list_of_ethernets(self):
+        url = reverse('ethernet-list')
+        response = self.client.get(url, format='json')
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data['count'], 2)
+
+    def test_get_ethernet_with_ip_details(self):
+        url = reverse('ethernet-detail', args=(self.eth.id,))
+        response = self.client.get(url, format='json')
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data['ipaddress'], {
+            'id': self.ip.id,
+            'address': self.ip.address,
+            'hostname': self.ip.hostname,
+            'dhcp_expose': self.ip.dhcp_expose,
+            'is_management': self.ip.is_management,
+            'url': self.get_full_url(
+                reverse('ipaddress-detail', args=(self.ip.id,))
+            )
+        })
 
     def test_cannot_delete_when_exposed_in_dhcp(self):
         url = reverse('ethernet-detail', args=(self.eth.id,))
@@ -769,3 +791,11 @@ def test_cannot_delete_when_exposed_in_dhcp(self):
             'Could not delete Ethernet when it is exposed in DHCP',
             response.data
         )
+
+    def test_filter_by_ipaddress(self):
+        url = '{}?ipaddress__address={}'.format(
+            reverse('ethernet-list'), self.ip.address
+        )
+        response = self.client.get(url, format='json')
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data['count'], 1)

src/ralph/data_center/admin.py

@@ -258,7 +258,8 @@ class DataCenterAssetAdmin(
     list_select_related = [
         'model', 'model__manufacturer', 'model__category', 'rack',
         'rack__server_room', 'rack__server_room__data_center', 'service_env',
-        'service_env__service', 'service_env__environment', 'configuration_path'
+        'service_env__service', 'service_env__environment',
+        'configuration_path__module',
     ]
     raw_id_fields = [
         'model', 'rack', 'service_env', 'parent', 'budget_info',

src/ralph/data_center/api/serializers.py

@@ -1,9 +1,12 @@
 # -*- coding: utf-8 -*-
-from rest_framework import serializers
+from rest_framework import fields, serializers
 
 from ralph.api import RalphAPISerializer
-from ralph.assets.api.serializers import AssetSerializer, BaseObjectSerializer
-from ralph.assets.models.components import Ethernet
+from ralph.assets.api.serializers import (
+    AssetSerializer,
+    BaseObjectSerializer,
+    EthernetSimpleSerializer
+)
 from ralph.data_center.models import (
     Accessory,
     BaseObjectCluster,
@@ -81,21 +84,27 @@ class Meta:
         exclude = ('accessories',)
 
 
-class EthernetSerializer(RalphAPISerializer):
-    class Meta:
-        model = Ethernet
-        depth = 1
-
-
+# used by DataCenterAsset and VirtualServer serializers
 class ComponentSerializerMixin(serializers.Serializer):
-    # ethernets = EthernetSerializer()
-    # disk_shares = ..
-    # ipaddresses = SimpleIPAddressesSerializer()
-    # mac_addresses = SimpleRackSerializer()
-    service = serializers.SerializerMethodField('get_service_env')
-
-    def get_service_env(self, obj):
-        return ''
+    ethernet = EthernetSimpleSerializer(many=True)
+    ipaddresses = fields.SerializerMethodField()
+
+    def get_ipaddresses(self, instance):
+        """
+        Return list of ip addresses for passed instance.
+
+        Returns:
+            list of ip addresses (as strings)
+        """
+        # don't use `ipaddresses` property here to make use of
+        # `ethernet__ipaddresses` in prefetch related
+        ipaddresses = []
+        for eth in instance.ethernet.all():
+            try:
+                ipaddresses.append(eth.ipaddress.address)
+            except AttributeError:
+                pass
+        return ipaddresses
 
 
 class RackSerializer(RalphAPISerializer):

src/ralph/data_center/api/views.py

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
-
+from django.db.models import Prefetch
 
 from ralph.api import RalphAPIViewSet
 from ralph.assets.api.filters import NetworkableObjectFilters
 from ralph.assets.api.views import BaseObjectViewSet, BaseObjectViewSetMixin
+from ralph.assets.models import Ethernet
 from ralph.data_center.admin import DataCenterAssetAdmin
 from ralph.data_center.api.serializers import (
     AccessorySerializer,
@@ -47,11 +48,12 @@ class DataCenterAssetViewSet(BaseObjectViewSetMixin, RalphAPIViewSet):
         'property_of', 'budget_info'
     ]
     prefetch_related = BaseObjectViewSet.prefetch_related + [
-        'service_env__service__environments',
-        'service_env__service__business_owners',
-        'service_env__service__technical_owners',
         'connections',
         'tags',
+        Prefetch(
+            'ethernet',
+            queryset=Ethernet.objects.select_related('ipaddress')
+        ),
     ]
     filter_fields = [
         'service_env__service__uid',

src/ralph/data_center/tests/factories.py

@@ -10,6 +10,8 @@
     BudgetInfoFactory,
     ConfigurationClassFactory,
     DataCenterAssetModelFactory,
+    EthernetFactory,
+    EthernetWithIPAddressFactory,
     ServiceEnvironmentFactory
 )
 from ralph.data_center.models.physical import (
@@ -115,6 +117,44 @@ class Meta:
         model = DataCenterAsset
 
 
+class DataCenterAssetFullFactory(DataCenterAssetFactory):
+    """
+    Factory for DataCenterAsset and m2m relations
+    """
+    rack = factory.SubFactory(RackFactory)
+
+    # m2m relations
+    # TODO: parent, networks (as terminators), operations, security scans,
+    # clusters, tags
+    eth1 = factory.RelatedFactory(
+        EthernetWithIPAddressFactory,
+        'base_object',
+        ipaddress__is_management=True,
+    )
+    eth2 = factory.RelatedFactory(EthernetFactory, 'base_object')
+    eth3 = factory.RelatedFactory(
+        EthernetWithIPAddressFactory,
+        'base_object',
+        ipaddress__dhcp_expose=True,
+    )
+    licence1 = factory.RelatedFactory(
+        'ralph.licences.tests.factories.BaseObjectLicenceFactory', 'base_object'
+    )
+    licence2 = factory.RelatedFactory(
+        'ralph.licences.tests.factories.BaseObjectLicenceFactory',
+        'base_object',
+        quantity=3
+    )
+    support1 = factory.RelatedFactory(
+        'ralph.supports.tests.factories.BaseObjectsSupportFactory',
+        'baseobject'
+    )
+    support2 = factory.RelatedFactory(
+        'ralph.supports.tests.factories.BaseObjectsSupportFactory',
+        'baseobject'
+    )
+
+
 class DatabaseFactory(DjangoModelFactory):
     service_env = factory.SubFactory(ServiceEnvironmentFactory)
 

src/ralph/data_center/tests/test_api.py

@@ -22,6 +22,7 @@
     ClusterFactory,
     ClusterTypeFactory,
     DataCenterAssetFactory,
+    DataCenterAssetFullFactory,
     RackAccessoryFactory,
     RackFactory,
     ServerRoomFactory
@@ -35,7 +36,7 @@ def setUp(self):
         self.service_env = ServiceEnvironmentFactory()
         self.model = DataCenterAssetModelFactory()
         self.rack = RackFactory()
-        self.dc_asset = DataCenterAssetFactory(
+        self.dc_asset = DataCenterAssetFullFactory(
             rack=self.rack,
             position=10,
             model=self.model,
@@ -44,11 +45,12 @@ def setUp(self):
             ethernet=EthernetFactory(base_object=self.dc_asset)
         )
         self.dc_asset.tags.add('db', 'test')
-        self.dc_asset_2 = DataCenterAssetFactory()
+        self.dc_asset_2 = DataCenterAssetFullFactory()
 
     def test_get_data_center_assets_list(self):
         url = reverse('datacenterasset-list')
-        response = self.client.get(url, format='json')
+        with self.assertNumQueries(9):
+            response = self.client.get(url, format='json')
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(
             response.data['count'], DataCenterAsset.objects.count()
@@ -65,6 +67,11 @@ def test_get_data_center_asset_details(self):
         self.assertEqual(
             response.data['model']['id'], self.dc_asset.model.id
         )
+        self.assertEqual(len(response.data['ethernet']), 4)
+        self.assertIn(self.ip.address, [
+            eth['ipaddress']['address'] for eth in response.data['ethernet']
+            if eth['ipaddress']
+        ])
 
     def test_create_data_center_asset(self):
         url = reverse('datacenterasset-list')

src/ralph/data_center/tests/test_models.py

@@ -225,17 +225,6 @@ def test_network_environment(self):
         self._prepare_rack(self.dc_asset, '192.168.1.0/24')
         self.assertEqual(self.dc_asset.network_environment, self.net_env)
 
-    def test_network_environment_many_options(self):
-        self._prepare_rack(self.dc_asset, '192.168.1.0/24')
-        proper_net_env = self.net_env
-        self._prepare_rack(self.dc_asset, '192.168.2.0/24', rack=self.rack)
-        self.assertGreater(
-            NetworkEnvironment.objects.filter(network__racks=self.rack).count(),
-            1
-        )
-        self.assertEqual(self.dc_asset.network_environment, proper_net_env)
-        self.assertNotEqual(self.dc_asset.network_environment, self.net_env)
-
     # =========================================================================
     # next free hostname
     # =========================================================================

src/ralph/licences/api.py

@@ -68,7 +68,9 @@ class BaseObjectLicenceViewSet(RalphAPIViewSet):
     serializer_class = BaseObjectLicenceSerializer
     select_related = [
         'licence', 'licence__region', 'licence__manufacturer',
-        'licence__licence_type', 'licence__software', 'base_object'
+        'licence__licence_type', 'licence__software', 'base_object',
+        'licence__budget_info', 'licence__office_infrastructure',
+        'licence__property_of',
     ]
     save_serializer_class = BaseObjectLicenceSerializer
 

src/ralph/licences/api_simple.py

@@ -10,7 +10,10 @@ class SimpleLicenceSerializer(RalphAPISerializer):
     class Meta:
         model = Licence
         depth = 1
-        exclude = ('base_objects', 'users', 'content_type', 'service_env')
+        exclude = (
+            'base_objects', 'users', 'content_type', 'service_env', 'parent',
+            'configuration_path', 'tags'
+        )
 
 
 class SimpleLicenceUserSerializer(RalphAPISerializer):

src/ralph/licences/models.py

@@ -231,6 +231,15 @@ class Licence(Regionalizable, AdminAbsoluteUrlMixin, BaseObject):
     objects_used_free_with_related = LicencesUsedFreeRelatedObjectsManager()
 
     def __str__(self):
+        return "{} x {} - {} ({})".format(
+            self.number_bought,
+            self.software.name,
+            self.invoice_date,
+            self.niw,
+        )
+
+    @cached_property
+    def autocomplete_str(self):
         return "{} ({} free) x {} - {} ({})".format(
             self.number_bought,
             self.free,

src/ralph/networks/api.py

@@ -41,8 +41,8 @@ class Meta:
 
 
 class IPAddressSerializer(RalphAPISerializer):
-    ethernet = EthernetSerializer()
     network = NetworkSimpleSerializer()
+    ethernet = EthernetSerializer()
 
     class Meta:
         model = IPAddress
@@ -80,7 +80,7 @@ class IPAddressViewSet(RalphAPIViewSet):
     ]
     filter_fields = [
         'hostname', 'ethernet__base_object', 'network', 'network__address',
-        'status', 'is_public', 'is_management', 'dhcp_expose'
+        'status', 'is_public', 'is_management', 'dhcp_expose', 'ethernet__mac',
     ]
 
     def destroy(self, request, *args, **kwargs):

src/ralph/networks/api_simple.py

@@ -0,0 +1,10 @@
+from ralph.api import RalphAPISerializer
+from ralph.networks.models import IPAddress
+
+
+class IPAddressSimpleSerializer(RalphAPISerializer):
+    class Meta:
+        model = IPAddress
+        fields = (
+            'id', 'address', 'hostname', 'dhcp_expose', 'is_management', 'url'
+        )