Merge pull request #3810 from allegro/optimize-hosts-endpoints

Optimize hosts endpoints
allegro · Jun 11, 2024 · 2931ba7 · 2931ba7
2 parents 5440253 + b62440b
commit 2931ba7
Show file tree

Hide file tree

Showing 15 changed files with 61 additions and 52 deletions.
diff --git a/src/ralph/assets/api/views.py b/src/ralph/assets/api/views.py
@@ -12,7 +12,6 @@
 from ralph.licences.api import BaseObjectLicenceViewSet
 from ralph.licences.models import BaseObjectLicence
 from ralph.networks.models import IPAddress
-from ralph.security.models import SecurityScan
 
 
 class BusinessSegmentViewSet(RalphAPIViewSet):
@@ -98,6 +97,8 @@ class Meta(NetworkableObjectFilters.Meta):
     Prefetch('licences', queryset=BaseObjectLicence.objects.select_related(
         *BaseObjectLicenceViewSet.select_related
     )),
+    'securityscan__tags',
+    'securityscan__vulnerabilities__tags',
     'custom_fields',
     'service_env__service__business_owners',
     'service_env__service__technical_owners',
@@ -244,23 +245,19 @@ class DCHostViewSet(BaseObjectViewSetMixin, RalphAPIViewSet):
         'content_type',
     ]
     select_related = [
-        'service_env', 'service_env__service', 'service_env__environment',
-        'configuration_path', 'configuration_path__module',
+        'service_env__service', 'service_env__environment',
+        'configuration_path__module',
         'parent__cloudproject',
     ]
     prefetch_related = [
         'tags',
         'custom_fields',
+        'securityscan__vulnerabilities__tags',
+        'securityscan__tags',
         Prefetch(
             'ethernet_set',
             queryset=models.Ethernet.objects.select_related('ipaddress')
         ),
-        Prefetch(
-            'securityscan',
-            queryset=SecurityScan.objects.prefetch_related(
-                'vulnerabilities', 'tags'
-            )
-        ),
     ]
     extended_filter_fields = {
         'name': [

diff --git a/src/ralph/assets/tests/test_api.py b/src/ralph/assets/tests/test_api.py
@@ -954,11 +954,14 @@ def setUp(self):
         self.cloud_host.update_custom_field('test_cf', 'xyz')
 
     def test_get_dc_hosts_list(self):
-        url = reverse('dchost-list')
-        with self.assertNumQueries(12):
+        dc_assets = DataCenterAssetFullFactory.create_batch(20)
+        VirtualServerFullFactory.create_batch(20, parent=dc_assets[0])
+        CloudHostFullFactory.create_batch(20, hypervisor=dc_assets[0])
+        url = reverse('dchost-list') + "?limit=100"
+        with self.assertNumQueries(15):
             response = self.client.get(url, format='json')
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(response.data['count'], 3)
+        self.assertEqual(response.data['count'], 63)
 
     def test_filter_by_type_dc_asset(self):
         url = '{}?{}'.format(

diff --git a/src/ralph/back_office/api.py b/src/ralph/back_office/api.py
@@ -59,6 +59,7 @@ class BackOfficeAssetViewSet(RalphAPIViewSet):
         'service_env__service__business_owners',
         'service_env__service__technical_owners',
         'tags',
+        'content_type',
     ]
     queryset = BackOfficeAsset.objects.all()
     serializer_class = BackOfficeAssetSerializer

diff --git a/src/ralph/back_office/tests/factories.py b/src/ralph/back_office/tests/factories.py
@@ -15,6 +15,7 @@
     OfficeInfrastructure,
     Warehouse
 )
+from ralph.security.tests.factories import SecurityScanFactory
 
 date_now = datetime.now().date()
 
@@ -57,6 +58,7 @@ class BackOfficeAssetFactory(DjangoModelFactory):
     invoice_date = date_now - timedelta(days=15)
     invoice_no = factory.Sequence(lambda n: 'Invoice number ' + str(n))
     price = FuzzyDecimal(10, 300)
+    securityscan = factory.RelatedFactory(SecurityScanFactory, 'base_object')
 
     class Meta:
         model = BackOfficeAsset

diff --git a/src/ralph/back_office/tests/test_api.py b/src/ralph/back_office/tests/test_api.py
@@ -30,8 +30,10 @@ def setUp(self):
         self.bo_asset.save()
 
     def test_get_back_office_assets_list(self):
-        url = reverse('backofficeasset-list')
-        response = self.client.get(url, format='json')
+        BackOfficeAssetFactory.create_batch(100)
+        url = reverse('backofficeasset-list') + "?limit=100"
+        with self.assertNumQueries(14):
+            response = self.client.get(url, format='json')
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(
             response.data['count'], BackOfficeAsset.objects.count()

diff --git a/src/ralph/data_center/api/views.py b/src/ralph/data_center/api/views.py
@@ -36,7 +36,6 @@
     ServerRoom,
     VIP
 )
-from ralph.security.models import SecurityScan
 
 
 class DataCenterAssetFilterSet(NetworkableObjectFilters):
@@ -64,12 +63,6 @@ class DataCenterAssetViewSet(BaseObjectViewSetMixin, RalphAPIViewSet):
             'ethernet_set',
             queryset=Ethernet.objects.select_related('ipaddress')
         ),
-        Prefetch(
-            'securityscan',
-            queryset=SecurityScan.objects.prefetch_related(
-                'vulnerabilities', 'tags'
-            )
-        ),
         'fibrechannelcard_set',
         'processor_set',
         'disk_set',

diff --git a/src/ralph/data_center/tests/factories.py b/src/ralph/data_center/tests/factories.py
@@ -38,6 +38,7 @@
     VIP,
     VIPProtocol
 )
+from ralph.security.tests.factories import SecurityScanFactory
 
 date_now = datetime.now().date()
 
@@ -70,9 +71,7 @@ class Meta:
 
 class ClusterFactory(DjangoModelFactory):
 
-    name = factory.Iterator(
-        ['Databases', 'Applications', 'Switch', 'Load balancer']
-    )
+    name = factory.Sequence(lambda n: f"Cluster {n}")
     type = factory.SubFactory(ClusterTypeFactory)
     configuration_path = factory.SubFactory(ConfigurationClassFactory)
     service_env = factory.SubFactory(ServiceEnvironmentFactory)
@@ -211,6 +210,8 @@ class DataCenterAssetFullFactory(DataCenterAssetFactory):
         'base_object',
     )
 
+    securityscan = factory.RelatedFactory(SecurityScanFactory, 'base_object')
+
     @factory.post_generation
     def post_tags(self, create, extracted, **kwargs):
         self.tags.add('abc, cde', 'xyz')

diff --git a/src/ralph/data_center/tests/test_api.py b/src/ralph/data_center/tests/test_api.py
@@ -54,8 +54,9 @@ def setUp(self):
         self.dc_asset_2 = DataCenterAssetFullFactory()
 
     def test_get_data_center_assets_list(self):
-        url = reverse('datacenterasset-list')
-        with self.assertNumQueries(17):
+        DataCenterAssetFullFactory.create_batch(100)
+        url = reverse('datacenterasset-list') + "?limit=100"
+        with self.assertNumQueries(20):
             response = self.client.get(url, format='json')
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(
@@ -551,18 +552,19 @@ def test_create_cluster_without_hostname_or_name(self):
         })
 
     def test_list_cluster(self):
-        url = reverse('cluster-list')
-        with self.assertNumQueries(12):
+        ClusterFactory.create_batch(20)
+        url = reverse('cluster-list') + "?limit=100"
+        with self.assertNumQueries(13):
             response = self.client.get(url, format='json')
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(len(response.data['results']), 2)
+        self.assertEqual(len(response.data['results']), 22)
         for item in response.data['results']:
             if item['id'] == self.cluster_1.id:
                 self.assertEqual(len(item['base_objects']), 2)
 
     def test_get_cluster_details(self):
         url = reverse('cluster-detail', args=(self.cluster_1.id,))
-        with self.assertNumQueries(11):
+        with self.assertNumQueries(12):
             response = self.client.get(url, format='json')
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response.data['name'], self.cluster_1.name)

diff --git a/src/ralph/data_center/tests/test_view.py b/src/ralph/data_center/tests/test_view.py
@@ -127,6 +127,7 @@ def setUp(self):
             rack__name='Rack #1',
             rack__server_room__name='SR1',
             rack__server_room__data_center__name='DC1',
+            securityscan=None,
         )
 
     def test_listing_show_ok_when_scan_succeed_and_no_vulnerabilities(
@@ -219,6 +220,7 @@ def setUp(self):
             rack__name='Rack #1',
             rack__server_room__name='SR1',
             rack__server_room__data_center__name='DC1',
+            securityscan=None,
         )
         self.scan_no_vuls = SecurityScanFactory(
             base_object=self.asset_no_vuls.baseobject_ptr, vulnerabilities=[],
@@ -232,6 +234,7 @@ def setUp(self):
             rack__name='Rack #1',
             rack__server_room__name='SR1',
             rack__server_room__data_center__name='DC1',
+            securityscan=None,
         )
         self.scan_with_vuls2 = SecurityScanFactory(
             base_object=self.asset_with_today_vul.baseobject_ptr,
@@ -246,6 +249,7 @@ def setUp(self):
            rack__name='Rack #1',
             rack__server_room__name='SR1',
             rack__server_room__data_center__name='DC1',
+            securityscan=None,
         )
         self.scan_with_vuls2 = SecurityScanFactory(
             base_object=self.asset_vuls2.baseobject_ptr,

diff --git a/src/ralph/security/api.py b/src/ralph/security/api.py
@@ -19,7 +19,7 @@ class Meta:
 
 
 class VulnerabilityViewSet(RalphAPIViewSet):
-    queryset = Vulnerability.objects.all()
+    queryset = Vulnerability.objects.all().prefetch_related('tags')
     serializer_class = VulnerabilitySerializer
     filter_fields = [
         'external_vulnerability_id',

diff --git a/src/ralph/security/tests/test_api.py b/src/ralph/security/tests/test_api.py
@@ -241,6 +241,16 @@ def test_get_vulnerability_list(self):
             response.data['count'], Vulnerability.objects.count()
         )
 
+    def test_get_vulnerability_list_query_count(self):
+        VulnerabilityFactory.create_batch(99)
+        url = reverse('vulnerability-list')
+        with self.assertNumQueries(5):
+            response = self.client.get(url + "?limit=100", format='json')
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(
+            response.data['count'], Vulnerability.objects.count()
+        )
+
     def test_get_vulnerability_details(self):
         url = reverse('vulnerability-detail', args=(self.vulnerability.id,))
         response = self.client.get(url, format='json')

diff --git a/src/ralph/virtual/api.py b/src/ralph/virtual/api.py
@@ -24,7 +24,6 @@
 from ralph.data_center.models import DCHost
 from ralph.lib.api.exceptions import Conflict
 from ralph.security.api import SecurityScanSerializer
-from ralph.security.models import SecurityScan
 from ralph.virtual.admin import VirtualServerAdmin
 from ralph.virtual.models import (
     CloudFlavor,
@@ -247,12 +246,6 @@ class CloudHostViewSet(BaseObjectViewSetMixin, RalphAPIViewSet):
             'ethernet_set',
             queryset=Ethernet.objects.select_related('ipaddress')
         ),
-        Prefetch(
-            'securityscan',
-            queryset=SecurityScan.objects.prefetch_related(
-                'vulnerabilities', 'tags'
-            )
-        ),
     ]
 
     filter_fields = [
@@ -298,12 +291,6 @@ class VirtualServerViewSet(BaseObjectViewSetMixin, RalphAPIViewSet):
             'ethernet_set',
             queryset=Ethernet.objects.select_related('ipaddress')
         ),
-        Prefetch(
-            'securityscan',
-            queryset=SecurityScan.objects.prefetch_related(
-                'vulnerabilities', 'tags'
-            )
-        ),
         # TODO: clusters
     ]
     filter_fields = [

diff --git a/src/ralph/virtual/tests/factories.py b/src/ralph/virtual/tests/factories.py
@@ -11,6 +11,7 @@
     ServiceEnvironmentFactory
 )
 from ralph.data_center.tests.factories import DataCenterAssetFactory
+from ralph.security.tests.factories import SecurityScanFactory
 from ralph.virtual.models import (
     CloudFlavor,
     CloudHost,
@@ -80,7 +81,7 @@ class CloudHostFactory(DjangoModelFactory):
         CloudProviderFactory,
         name='openstack',
     )
-    host_id = factory.Iterator(['host_id1', 'host_id2', 'host_id3', 'host_id4'])
+    host_id = factory.Sequence(lambda n: f'host_id1{n}.local')
     parent = factory.SubFactory(CloudProjectFactory)
     configuration_path = factory.SubFactory(ConfigurationClassFactory)
     service_env = factory.SubFactory(ServiceEnvironmentFactory)
@@ -92,6 +93,8 @@ class Meta:
 
 class CloudHostFullFactory(CloudHostFactory):
     hypervisor = factory.SubFactory(DataCenterAssetFactory)
+    securityscan = factory.RelatedFactory(SecurityScanFactory, 'base_object')
+
 
     @factory.post_generation
     def post_tags(self, create, extracted, **kwargs):
@@ -141,6 +144,7 @@ class VirtualServerFullFactory(VirtualServerFactory):
     proc2 = factory.RelatedFactory(ProcessorFactory, 'base_object')
     disk1 = factory.RelatedFactory(DiskFactory, 'base_object')
     disk2 = factory.RelatedFactory(DiskFactory, 'base_object')
+    securityscan = factory.RelatedFactory(SecurityScanFactory, 'base_object')
 
     @factory.post_generation
     def post_tags(self, create, extracted, **kwargs):

diff --git a/src/ralph/virtual/tests/test_api.py b/src/ralph/virtual/tests/test_api.py
@@ -29,6 +29,7 @@
 from ralph.virtual.tests.factories import (
     CloudFlavorFactory,
     CloudHostFactory,
+    CloudHostFullFactory,
     CloudProjectFactory,
     CloudProviderFactory,
     VirtualServerFullFactory
@@ -97,11 +98,12 @@ def test_get_list(self, field):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
 
     def test_get_cloudhost_list(self):
-        url = reverse('cloudhost-list')
-        with self.assertNumQueries(12):
+        CloudHostFullFactory.create_batch(100)
+        url = reverse('cloudhost-list') + "?limit=100"
+        with self.assertNumQueries(15):
             response = self.client.get(url, format='json')
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(response.data['count'], 2)
+        self.assertEqual(response.data['count'], 102)
 
     @unpack
     @data(
@@ -450,11 +452,12 @@ def setUp(self):
         )
 
     def test_get_virtual_server_list(self):
-        url = reverse('virtualserver-list')
-        with self.assertNumQueries(13):
+        VirtualServerFullFactory.create_batch(20)
+        url = reverse('virtualserver-list') + "?limit=100"
+        with self.assertNumQueries(16):
             response = self.client.get(url, format='json')
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(response.data['count'], 2)
+        self.assertEqual(response.data['count'], 22)
 
     def test_get_virtual_server_details(self):
         url = reverse('virtualserver-detail', args=(self.virtual_server.id,))

diff --git a/src/ralph/virtual/tests/test_models.py b/src/ralph/virtual/tests/test_models.py
@@ -249,7 +249,7 @@ def test_cleanup_security_scan_transition(self):
 
 class VirtualServerTestCase(RalphTestCase, NetworkableBaseObjectTestMixin):
     def setUp(self):
-        self.vs = VirtualServerFullFactory()
+        self.vs = VirtualServerFullFactory(securityscan=None)
         self.custom_field_str = CustomField.objects.create(
             name='test str', type=CustomFieldTypes.STRING, default_value='xyz'
         )