fix: Rewritten AzureVirtualMachineSize collecting

plugins/azure/resoto_plugin_azure/collector.py

@@ -1,15 +1,15 @@
 import logging
 from concurrent.futures import ThreadPoolExecutor, Future
-from typing import Type, Set, List
-
+from typing import Any, Optional, Type, Set, List
 
 from resoto_plugin_azure.config import AzureConfig, AzureCredentials
 from resoto_plugin_azure.azure_client import AzureClient
 from resoto_plugin_azure.resource.compute import (
-    filter_resources as network_filter_resources,
+    AzureVirtualMachineSize,
     resources as compute_resources,
 )
 from resoto_plugin_azure.resource.base import (
+    AzureLocation,
     AzureSubscription,
     GraphBuilder,
     AzureResource,
@@ -82,7 +82,7 @@ def collect(self) -> None:
             # wait for all work to finish
             queue.wait_for_submitted_work()
             # filter nodes
-            self.filter_nodes(builder)
+            self.filter_nodes()
             self.core_feedback.progress_done(self.subscription.subscription_id, 1, 1, context=[self.cloud.id])
             log.info(f"[Azure:{self.subscription.safe_name}] Collecting resources done.")
 
@@ -105,14 +105,19 @@ def work_done(_: Future[None]) -> None:
         all_done.add_done_callback(work_done)
         return all_done
 
-    def filter_nodes(self, builder: GraphBuilder) -> None:
-        resources_to_remove = network_filter_resources
+    def filter_nodes(self) -> None:
         remove_nodes = []
 
-        def rm_nodes(cls: Type[AzureResource]) -> None:
-            to_compare = cls.collect_resources_for_comparison(builder)
+        def rm_nodes(cls, ignore_kinds: Optional[Type[Any]] = None) -> None:
             for node in self.graph.nodes:
-                if isinstance(node, cls) and node.filter_node(to_compare):
+                if not isinstance(node, cls):
+                    continue
+                pred = list(self.graph.predecessors(node))
+                if ignore_kinds is not None:
+                    pred = [p for p in pred if not isinstance(p, ignore_kinds)]
+
+                if not pred:
+                    remove_nodes.extend(pred)
                     remove_nodes.append(node)
             removed = set()
             for node in remove_nodes:
@@ -123,5 +128,4 @@ def rm_nodes(cls: Type[AzureResource]) -> None:
             log.debug(f"Removing {len(remove_nodes)} unreferenced nodes of type {cls}")
             remove_nodes.clear()
 
-        for resource_class in resources_to_remove:
-            rm_nodes(resource_class)
+        rm_nodes(AzureVirtualMachineSize, AzureLocation)

plugins/azure/resoto_plugin_azure/resource/base.py

@@ -134,32 +134,8 @@ def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None:
         # Default behavior: add resource to the namespace
         pass
 
-    def filter_node(self, compare_resources: Optional[List[Tuple[str, str]]]) -> bool:
-        """
-        Determine whether the current instance meets exclusion criteria based on a list of resource tuples for comparison.
-
-        Parameters:
-        - compare_resources (Optional[List[Tuple[str, str]]]): A list of resource tuples for comparison.
-
-        Returns:
-        bool: True if the instance meets exclusion criteria; False if it does not.
-
-        Default: return False. It means that, by default, the node does not need to be excluded.
-        """
-        return False
-
-    @classmethod
-    def collect_resources_for_comparison(
-        cls: Type[AzureResourceType], builder: GraphBuilder
-    ) -> Optional[List[Tuple[str, str]]]:
-        """
-        Perform any pre-collection steps and return a list of items for comparison if needed.
-        """
-        return None
-
-    @classmethod
     def fetch_resources(
-        cls: Type[AzureResourceType],
+        self,
         builder: GraphBuilder,
         service: str,
         api_version: str,

plugins/azure/resoto_plugin_azure/resource/compute.py

@@ -1,12 +1,11 @@
 from datetime import datetime
-from typing import ClassVar, Dict, Optional, List, Any, Tuple, Type
+from typing import ClassVar, Dict, Optional, List, Any, Type
 
 from attr import define, field
 
 from resoto_plugin_azure.azure_client import AzureApiSpec
 from resoto_plugin_azure.resource.base import (
     AzureResource,
-    AzureResourceType,
     GraphBuilder,
     AzureSubResource,
     AzureSystemData,
@@ -2530,11 +2529,7 @@ class AzureVirtualMachine(AzureResource, BaseInstance):
             ]
         },
         "successors": {
-            "default": [
-                "azure_image",
-                "azure_disk",
-                "azure_network_interface",
-            ]
+            "default": ["azure_image", "azure_disk", "azure_network_interface", "azure_virtual_machine_size"]
         },
     }
     mapping: ClassVar[Dict[str, Bender]] = {
@@ -2669,6 +2664,10 @@ def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None:
             for network_interface in network_interfaces:
                 if ni_id := network_interface.id:
                     builder.add_edge(self, edge_type=EdgeType.default, clazz=AzureNetworkInterface, id=ni_id)
+        if (vms_type := self.instance_type) and (vm_location := self.location):
+            builder.add_edge(
+                self, edge_type=EdgeType.default, clazz=AzureVirtualMachineSize, name=vms_type, location=vm_location
+            )
 
 
 @define(eq=False, slots=False)
@@ -3214,7 +3213,7 @@ class AzureVirtualMachineSize(AzureResource, BaseInstanceType):
         expect_array=True,
     )
     mapping: ClassVar[Dict[str, Bender]] = {
-        "id": K(None),
+        "id": S("name"),
         "tags": S("tags", default={}),
         "name": S("name"),
         "ctime": K(None),
@@ -3234,38 +3233,10 @@ class AzureVirtualMachineSize(AzureResource, BaseInstanceType):
     number_of_cores: Optional[int] = field(default=None, metadata={'description': 'The number of cores supported by the virtual machine size. For constrained vcpu capable vm sizes, this number represents the total vcpus of quota that the vm uses. For accurate vcpu count, please refer to https://docs. Microsoft. Com/azure/virtual-machines/constrained-vcpu or https://docs. Microsoft. Com/rest/api/compute/resourceskus/list.'})  # fmt: skip
     os_disk_size_in_mb: Optional[int] = field(default=None, metadata={'description': 'The os disk size, in mb, allowed by the virtual machine size.'})  # fmt: skip
     resource_disk_size_in_mb: Optional[int] = field(default=None, metadata={'description': 'The resource disk size, in mb, allowed by the virtual machine size.'})  # fmt: skip
-    _location: Optional[str] = field(default=None, metadata={"description": "Resource location."})
+    location: Optional[str] = field(default=None, metadata={"description": "Resource location."})
 
     def pre_process(self, graph_builder: GraphBuilder, source: Json) -> None:
-        self._location = graph_builder.location.name if graph_builder.location else ""
-
-    def filter_node(self, compare_resources: Optional[List[Tuple[str, str]]]) -> bool:
-        if compare_resources is not None:
-            vm_type_and_location = compare_resources
-            vm_size_type = self.instance_type if self.instance_type else ""
-            vm_size_location = self._location if self._location else ""
-            for vm_info in vm_type_and_location:
-                vm_type, vm_location = vm_info
-                if vm_size_type == vm_type and vm_size_location == vm_location:
-                    return False
-        return True
-
-    @classmethod
-    def collect_resources_for_comparison(
-        cls: Type[AzureResourceType], builder: GraphBuilder
-    ) -> Optional[List[Tuple[str, str]]]:
-        virtual_machines_info = cls.fetch_resources(
-            builder,
-            service="compute",
-            api_version="2023-03-01",
-            path="/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachines",
-            path_parameters=["subscriptionId"],
-            query_parameters=["api-version"],
-            first_property=lambda r: r["properties"]["hardwareProfile"]["vmSize"],
-            second_property=lambda r: r["location"],
-        )
-
-        return [(str(vm_type), str(vm_location)) for vm_type, vm_location in virtual_machines_info]
+        self.location = graph_builder.location.name if graph_builder.location else ""
 
 
 resources: List[Type[AzureResource]] = [
@@ -3288,5 +3259,3 @@ def collect_resources_for_comparison(
     AzureVirtualMachineScaleSet,
     AzureVirtualMachineSize,
 ]
-
-filter_resources: List[Type[AzureResource]] = [AzureVirtualMachineSize]

plugins/azure/resoto_plugin_azure/resource/network.py

@@ -1603,8 +1603,8 @@ def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None:
             for ip_configuration in ip_configurations:
                 if subnet_id := ip_configuration.subnet:
                     builder.add_edge(self, edge_type=EdgeType.default, reverse=True, clazz=AzureSubnet, id=subnet_id)
-                if p_ip_address := ip_configuration.public_ip_address:
-                    builder.add_edge(self, edge_type=EdgeType.default, clazz=AzurePublicIPAddress, id=p_ip_address)
+                if p_ip_address_id := ip_configuration.public_ip_address:
+                    builder.add_edge(self, edge_type=EdgeType.default, clazz=AzurePublicIPAddress, id=p_ip_address_id)
 
 
 @define(eq=False, slots=False)
@@ -1702,8 +1702,8 @@ def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None:
             for vn_id in vns:
                 builder.add_edge(self, edge_type=EdgeType.default, clazz=AzureVirtualNetwork, id=vn_id)
         if p_ip_addresses := self.public_ip_addresses:
-            for p_ip_address in p_ip_addresses:
-                builder.add_edge(self, edge_type=EdgeType.default, clazz=AzurePublicIPAddress, id=p_ip_address)
+            for p_ip_address_id in p_ip_addresses:
+                builder.add_edge(self, edge_type=EdgeType.default, clazz=AzurePublicIPAddress, id=p_ip_address_id)
 
 
 @define(eq=False, slots=False)
@@ -3066,7 +3066,6 @@ class AzureExpressRouteCircuit(AzureResource):
         expect_array=True,
     )
     reference_kinds: ClassVar[ModelReference] = {
-        "predecessors": {"default": ["azure_virtual_network"]},
         "successors": {"default": ["azure_express_route_port", "azure_express_route_ports_location"]},
     }
     mapping: ClassVar[Dict[str, Bender]] = {
@@ -3141,16 +3140,6 @@ def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None:
                     builder.add_edge(
                         self, edge_type=EdgeType.default, clazz=AzureExpressRoutePortsLocation, id=erplocation_id
                     )
-        if c_peerings := self.circuit_peerings:
-            for c_peering in c_peerings:
-                if p_address := c_peering.primary_peer_address_prefix:
-                    builder.add_edge(
-                        self, edge_type=EdgeType.default, reverse=True, clazz=AzureVirtualNetwork, id=p_address
-                    )
-                if s_address := c_peering.secondary_peer_address_prefix:
-                    builder.add_edge(
-                        self, edge_type=EdgeType.default, reverse=True, clazz=AzureVirtualNetwork, id=s_address
-                    )
 
 
 @define(eq=False, slots=False)
@@ -4339,7 +4328,6 @@ class AzureNetworkVirtualAppliance(AzureResource):
         expect_array=True,
     )
     reference_kinds: ClassVar[ModelReference] = {
-        "predecessors": {"default": ["azure_virtual_network"]},
         "successors": {"default": ["azure_network_virtual_appliance_sku"]},
     }
     mapping: ClassVar[Dict[str, Bender]] = {
@@ -4411,13 +4399,7 @@ def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None:
                 vendor_name, nvasku_name = info
                 if vendor_name == nva_vendor:
                     builder.add_edge(
-                        self, edge_type=EdgeType.default, clazz=AzureNetworkVirtualApplianceSku, id=nvasku_name
-                    )
-        if virtual_appliances := self.virtual_appliance_nics:
-            for va in virtual_appliances:
-                if private_address := va.private_ip_address:
-                    builder.add_edge(
-                        self, edge_type=EdgeType.default, reverse=True, clazz=AzureVirtualNetwork, id=private_address
+                        self, edge_type=EdgeType.default, clazz=AzureNetworkVirtualApplianceSku, name=nvasku_name
                     )
 
 

plugins/azure/test/collector_test.py

@@ -16,4 +16,4 @@ def test_collect(
     collector = AzureSubscriptionCollector(config, Cloud(id="azure"), azure_subscription, credentials, core_feedback)
     collector.collect()
     assert len(collector.graph.nodes) == 418
-    assert len(collector.graph.edges) == 471
+    assert len(collector.graph.edges) == 473

plugins/azure/test/compute_test.py

@@ -124,6 +124,7 @@ def test_virtual_machine(builder: GraphBuilder) -> None:
         AzureNetworkInterface,
         AzureNetworkSecurityGroup,
         AzureLoadBalancer,
+        AzureVirtualMachineSize,
     ]
     connect_resources(builder, resource_types)
 
@@ -146,6 +147,18 @@ def test_scale_set(builder: GraphBuilder) -> None:
     assert len(collected) == 1
 
 
+def test_virtual_machine_size(builder: GraphBuilder) -> None:
+    collected = roundtrip_check(AzureVirtualMachineSize, builder)
+    assert len(collected) == 12
+
+
+def test_virtual_machine_size_resources(builder: GraphBuilder) -> None:
+    collected = roundtrip_check(AzureVirtualMachineSize, builder)[0]
+    assert collected.instance_type == "Standard_A1_V2"
+    assert collected.instance_cores == 1.0
+    assert collected.instance_memory == 2.0
+
+
 def test_snapshot(builder: GraphBuilder) -> None:
     collected = roundtrip_check(AzureSnapshot, builder)
     assert len(collected) == 2

plugins/azure/test/files/compute/vmSizes.json

@@ -15,6 +15,86 @@
             "resourceDiskSizeInMB": 20480,
             "memoryInMB": 4096,
             "maxDataDiskCount": 4
+        },
+        {
+            "name": "Standard_B1s",
+            "numberOfCores": 1,
+            "osDiskSizeInMB": 1047552,
+            "resourceDiskSizeInMB": 5120,
+            "memoryInMB": 2048,
+            "maxDataDiskCount": 1
+        },
+        {
+            "name": "Standard_D2s_v3",
+            "numberOfCores": 2,
+            "osDiskSizeInMB": 1047552,
+            "resourceDiskSizeInMB": 20480,
+            "memoryInMB": 8192,
+            "maxDataDiskCount": 4
+        },
+        {
+            "name": "Standard_F4s",
+            "numberOfCores": 4,
+            "osDiskSizeInMB": 1047552,
+            "resourceDiskSizeInMB": 40960,
+            "memoryInMB": 16384,
+            "maxDataDiskCount": 8
+        },
+        {
+            "name": "Standard_DS1_v2",
+            "numberOfCores": 1,
+            "osDiskSizeInMB": 1047552,
+            "resourceDiskSizeInMB": 10240,
+            "memoryInMB": 3584,
+            "maxDataDiskCount": 2
+        },
+        {
+            "name": "Standard_E2s_v4",
+            "numberOfCores": 2,
+            "osDiskSizeInMB": 1047552,
+            "resourceDiskSizeInMB": 20480,
+            "memoryInMB": 8192,
+            "maxDataDiskCount": 4
+        },
+        {
+            "name": "Standard_G1",
+            "numberOfCores": 2,
+            "osDiskSizeInMB": 1047552,
+            "resourceDiskSizeInMB": 51200,
+            "memoryInMB": 8192,
+            "maxDataDiskCount": 8
+        },
+        {
+            "name": "Standard_H8",
+            "numberOfCores": 8,
+            "osDiskSizeInMB": 1047552,
+            "resourceDiskSizeInMB": 102400,
+            "memoryInMB": 65536,
+            "maxDataDiskCount": 16
+        },
+        {
+            "name": "Standard_A4m_v2",
+            "numberOfCores": 4,
+            "osDiskSizeInMB": 1047552,
+            "resourceDiskSizeInMB": 40960,
+            "memoryInMB": 16384,
+            "maxDataDiskCount": 8
+        },
+        {
+            "name": "Standard_NC6",
+            "numberOfCores": 6,
+            "osDiskSizeInMB": 1047552,
+            "resourceDiskSizeInMB": 81920,
+            "memoryInMB": 57344,
+            "maxDataDiskCount": 12
+        },
+        {
+            "name": "Standard_NV12",
+            "numberOfCores": 12,
+            "osDiskSizeInMB": 1047552,
+            "resourceDiskSizeInMB": 102400,
+            "memoryInMB": 57344,
+            "maxDataDiskCount": 24
         }
-    ]
+    ]    
 }