[azure][fix] Compute unused regions as last step (#2228)

Author: aquamatthias

plugins/aws/fix_plugin_aws/collector.py

@@ -366,7 +366,7 @@ def rm_leaf_nodes(cls: Any, ignore_kinds: Optional[Type[Any]] = None, check_pred
 
         rm_leaf_nodes(bedrock.AwsBedrockFoundationModel, check_pred=False)
         # remove regions that are not in use
-        self.graph.remove_recursively(builder.nodes(AwsRegion, lambda r: r.region_in_use is False))
+        self.graph.remove_recursively(builder.nodes(AwsRegion, lambda r: r.compute_region_in_use(builder) is False))
 
     # TODO: move into separate AwsAccountSettings
     def update_account(self) -> None:

plugins/aws/fix_plugin_aws/resource/base.py

@@ -370,7 +370,7 @@ def __attrs_post_init__(self) -> None:
         self.latitude = cloud_region_data.get("aws", {}).get(self.id, {}).get("latitude")
         self.longitude = cloud_region_data.get("aws", {}).get(self.id, {}).get("longitude")
 
-    def complete_graph(self, builder: GraphBuilder, source: Json) -> None:
+    def compute_region_in_use(self, builder: GraphBuilder) -> bool:
         count = 0
         ignore_kinds = {
             "aws_athena_work_group",
@@ -399,7 +399,9 @@ def ignore_for_count(resource: BaseResource) -> bool:
                 count += 1
                 if count > empty_region:
                     break
-        self.region_in_use = count > empty_region
+        in_use = count > empty_region
+        self.region_in_use = in_use
+        return in_use
 
 
 @define(eq=False, slots=False)

plugins/azure/fix_plugin_azure/collector.py

@@ -277,7 +277,7 @@ def remove_usage_zero_value() -> None:
         rm_leaf_nodes(AzureCosmosDBLocation, AzureLocation, check_pred=False)
         rm_leaf_nodes(AzureLocation, check_pred=False)
         remove_usage_zero_value()
-        self.graph.remove_recursively(builder.nodes(AzureLocation, lambda r: r.region_in_use is False))
+        self.graph.remove_recursively(builder.nodes(AzureLocation, lambda r: r.compute_region_in_use(builder) is False))
 
     def after_collect(self, builder: GraphBuilder) -> None:
         # Filter unnecessary nodes such as AzureComputeDiskTypePricing

plugins/azure/fix_plugin_azure/resource/base.py

@@ -342,7 +342,7 @@ class AzureLocation(MicrosoftResource, BaseRegion):
     regional_display_name: Optional[str] = field(default=None, metadata={'description': 'The display name of the location and its region.'})  # fmt: skip
     subscription_id: Optional[str] = field(default=None, metadata={"description": "The subscription id."})
 
-    def post_process(self, graph_builder: GraphBuilder, source: Json) -> None:
+    def compute_region_in_use(self, graph_builder: GraphBuilder) -> bool:
         ignore_kinds: Set[str] = {"azure_network_virtual_network", "azure_network_watcher"}
 
         def ignore_for_count(resource: BaseResource) -> bool:
@@ -361,7 +361,10 @@ def ignore_for_count(resource: BaseResource) -> bool:
                 count += 1
                 if count > empty_region:
                     break
-        self.region_in_use = count > empty_region
+
+        in_use = count > empty_region
+        self.region_in_use = in_use
+        return in_use
 
 
 @define(eq=False, slots=False)

plugins/azure/test/collector_test.py

@@ -48,8 +48,8 @@ def test_collect(
         config, Cloud(id="azure"), azure_subscription, credentials, core_feedback
     )
     subscription_collector.collect()
-    assert len(subscription_collector.graph.nodes) == 455
-    assert len(subscription_collector.graph.edges) == 679
+    assert len(subscription_collector.graph.nodes) == 588
+    assert len(subscription_collector.graph.edges) == 977
 
     graph_collector = MicrosoftGraphOrganizationCollector(
         config, Cloud(id="azure"), MicrosoftGraphOrganization(id="test", name="test"), credentials, core_feedback

plugins/gcp/fix_plugin_gcp/collector.py

@@ -140,7 +140,7 @@ def rm_leaf_nodes(clazz: Any, ignore_kinds: Optional[Type[Any]] = None) -> None:
         rm_leaf_nodes(billing.GcpSku)
         rm_leaf_nodes(billing.GcpService)
         # remove regions that are not in use
-        self.graph.remove_recursively(builder.nodes(GcpRegion, lambda r: r.region_in_use is False))
+        self.graph.remove_recursively(builder.nodes(GcpRegion, lambda r: r.compute_region_in_use(builder) is False))
 
     def collect_region(self, regional_builder: GraphBuilder) -> None:
         # fetch all region level resources

plugins/gcp/fix_plugin_gcp/resources/base.py

@@ -569,6 +569,7 @@ def post_process(self, graph_builder: GraphBuilder, source: Json) -> None:
             graph_builder.add_node(region_quota, source)
             graph_builder.add_edge(self, node=region_quota)
 
+    def compute_region_in_use(self, graph_builder: GraphBuilder) -> bool:
         ignore_kinds = {
             "gcp_subnetwork",  # There are subnetworks that are created by GCP automatically.
         }
@@ -589,7 +590,10 @@ def ignore_for_count(resource: BaseResource) -> bool:
                 count += 1
                 if count > empty_region:
                     break
-        self.region_in_use = count > empty_region
+
+        in_use = count > empty_region
+        self.region_in_use = in_use
+        return in_use
 
     def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None:
         super().connect_in_graph(builder, source)