[aws][feat] Make a collection of Ec2 Instance types only for existing instances (#2264)

1101-1 · aquamatthias · web-flow · commit bc8eae4ad003 · 2024-11-01T11:28:46.000+01:00
Co-authored-by: Matthias Veit &lt;matthias_veit@yahoo.de&gt;
diff --git a/plugins/aws/fix_plugin_aws/resource/base.py b/plugins/aws/fix_plugin_aws/resource/base.py
@@ -198,7 +198,7 @@ def from_api(cls: Type[AwsResourceType], json: Json, builder: GraphBuilder) -> O
         return parse_json(json, cls, builder, cls.mapping)
 
     @classmethod
-    def collect_resources(cls: Type[AwsResource], builder: GraphBuilder) -> None:
+    def collect_resources(cls, builder: GraphBuilder) -> None:
         # Default behavior: in case the class has an ApiSpec, call the api and call collect.
         log.debug(f"Collecting {cls.__name__} in region {builder.region.name}")
         if spec := cls.api_spec:
diff --git a/plugins/aws/fix_plugin_aws/resource/ec2.py b/plugins/aws/fix_plugin_aws/resource/ec2.py
@@ -6,6 +6,7 @@
 from functools import partial
 from typing import ClassVar, Dict, Optional, List, Type, Any
 
+from boto3.exceptions import Boto3Error
 from attrs import define, field
 
 from fix_plugin_aws.aws_client import AwsClient
@@ -22,6 +23,7 @@
 from fix_plugin_aws.resource.kms import AwsKmsKey
 from fix_plugin_aws.resource.s3 import AwsS3Bucket
 from fix_plugin_aws.utils import ToDict, TagsValue
+from fix_plugin_aws.aws_client import AwsClient
 from fixlib.baseresources import (
     BaseInstance,
     BaseKeyPair,
@@ -384,14 +386,14 @@ class AwsEc2InferenceAcceleratorInfo:
 
 @define(eq=False, slots=False)
 class AwsEc2InstanceType(AwsResource, BaseInstanceType):
+    # collected via AwsEc2Instance
     kind: ClassVar[str] = "aws_ec2_instance_type"
     _kind_display: ClassVar[str] = "AWS EC2 Instance Type"
     _kind_description: ClassVar[str] = "AWS EC2 Instance Types are predefined virtual server configurations offered by Amazon Web Services. Each type specifies the compute, memory, storage, and networking capacity of the virtual machine. Users select an instance type based on their application's requirements, balancing performance and cost. EC2 instances can be launched, stopped, and terminated as needed for various computing workloads."  # fmt: skip
     _docs_url: ClassVar[str] = "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html"
     _kind_service: ClassVar[Optional[str]] = service_name
     _metadata: ClassVar[Dict[str, Any]] = {"icon": "type", "group": "compute"}
     _aws_metadata: ClassVar[Dict[str, Any]] = {"arn_tpl": "arn:{partition}:ec2:{region}:{account}:instance/{id}"}  # fmt: skip
-    api_spec: ClassVar[AwsApiSpec] = AwsApiSpec(service_name, "describe-instance-types", "InstanceTypes")
     _reference_kinds: ClassVar[ModelReference] = {
         "successors": {
             "default": ["aws_ec2_instance"],
@@ -456,6 +458,29 @@ class AwsEc2InstanceType(AwsResource, BaseInstanceType):
     auto_recovery_supported: Optional[bool] = field(default=None)
     supported_boot_modes: List[str] = field(factory=list)
 
+    @classmethod
+    def collect_resource_types(cls, builder: GraphBuilder, instance_types: List[str]) -> None:
+        spec = AwsApiSpec(service_name, "describe-instance-types", "InstanceTypes")
+        log.debug(f"Collecting {cls.__name__} in region {builder.region.name}")
+        try:
+            filters = [{"Name": "instance-type", "Values": instance_types}]
+            items = builder.client.list(
+                aws_service=spec.service,
+                action=spec.api_action,
+                result_name=spec.result_property,
+                expected_errors=spec.expected_errors,
+                Filters=filters,
+            )
+            cls.collect(items, builder)
+        except Boto3Error as e:
+            msg = f"Error while collecting {cls.__name__} in region {builder.region.name}: {e}"
+            builder.core_feedback.error(msg, log)
+            raise
+        except Exception as e:
+            msg = f"Error while collecting {cls.__name__} in region {builder.region.name}: {e}"
+            builder.core_feedback.info(msg, log)
+            raise
+
     @classmethod
     def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None:
         for js in json:
@@ -467,6 +492,14 @@ def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) ->
                 # we collect instance types in all regions and make the data unique in the builder
                 builder.global_instance_types[it.safe_name] = it
 
+    @classmethod
+    def service_name(cls) -> Optional[str]:
+        return service_name
+
+    @classmethod
+    def called_collect_apis(cls) -> List[AwsApiSpec]:
+        return [AwsApiSpec(service_name, "describe-instance-types")]
+
 
 # endregion
 
@@ -1375,6 +1408,17 @@ class AwsEc2Instance(EC2Taggable, AwsResource, BaseInstance):
     instance_maintenance_options: Optional[str] = field(default=None)
     instance_user_data: Optional[str] = field(default=None)
 
+    @classmethod
+    def collect_resources(cls, builder: GraphBuilder) -> None:
+        super().collect_resources(builder)
+        ec2_instance_types = set()
+        for instance in builder.nodes(clazz=AwsEc2Instance):
+            ec2_instance_types.add(instance.instance_type)
+        if ec2_instance_types:
+            builder.submit_work(
+                service_name, AwsEc2InstanceType.collect_resource_types, builder, list(ec2_instance_types)
+            )
+
     @classmethod
     def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None:
         def fetch_user_data(instance: AwsEc2Instance) -> None:
diff --git a/plugins/aws/fix_plugin_aws/resource/iam.py b/plugins/aws/fix_plugin_aws/resource/iam.py
@@ -676,11 +676,11 @@ def called_collect_apis(cls) -> List[AwsApiSpec]:
         ]
 
     @classmethod
-    def collect_resources(cls: Type[AwsResource], builder: GraphBuilder) -> None:
+    def collect_resources(cls, builder: GraphBuilder) -> None:
         # start generation of the credentials resport and pick it up later
         builder.client.get(service_name, "generate-credential-report")
         # let super handle the rest (this will take some time for the report to be done)
-        super().collect_resources(builder)  # type: ignore # mypy bug: https://github.com/python/mypy/issues/12885
+        super().collect_resources(builder)
 
     @classmethod
     def collect(cls: Type[AwsResource], json_list: List[Json], builder: GraphBuilder) -> None:
diff --git a/plugins/aws/test/graphbuilder_test.py b/plugins/aws/test/graphbuilder_test.py
@@ -17,6 +17,7 @@ def test_instance_type(builder: GraphBuilder) -> None:
         cloud_instance_data, ["aws", instance_type, "pricing", builder.region.id, "linux", "ondemand"]
     )
     eu_builder = builder.for_region(AwsRegion(id="eu-central-1"))
+    builder.global_instance_types[instance_type] = AwsEc2InstanceType(id=instance_type)
     m4l_eu: AwsEc2InstanceType = eu_builder.instance_type(eu_builder.region, instance_type)  # type: ignore
     assert m4l != m4l_eu
     assert m4l_eu == eu_builder.instance_type(eu_builder.region, instance_type)
diff --git a/plugins/aws/test/resources/__init__.py b/plugins/aws/test/resources/__init__.py
@@ -180,7 +180,8 @@ def round_trip_for(
     to_collect = [cls] + collect_also if collect_also else [cls]
     builder = build_graph(to_collect, region_name=region_name)
     assert len(builder.graph.nodes) > 0
-    for node, data in builder.graph.nodes(data=True):
+    nodes_to_process = list(builder.graph.nodes(data=True))
+    for node, data in nodes_to_process:
         node.connect_in_graph(builder, data.get("source", {}))
         check_single_node(node)
     first = next(iter(builder.resources_of(cls)))
diff --git a/plugins/aws/test/resources/files/ec2/describe-instance-types__instance_type_m4_large_mac2_metal.json b/plugins/aws/test/resources/files/ec2/describe-instance-types__instance_type_m4_large_mac2_metal.json
@@ -0,0 +1,128 @@
+{
+    "InstanceTypes": [
+        {
+            "InstanceType": "m4.large",
+            "CurrentGeneration": true,
+            "FreeTierEligible": false,
+            "SupportedUsageClasses": [
+                "on-demand",
+                "spot"
+            ],
+            "SupportedRootDeviceTypes": [
+                "ebs"
+            ],
+            "SupportedVirtualizationTypes": [
+                "hvm"
+            ],
+            "BareMetal": false,
+            "Hypervisor": "nitro",
+            "ProcessorInfo": {
+                "SupportedArchitectures": [
+                    "x86_64"
+                ],
+                "SustainedClockSpeedInGhz": 3.5
+            },
+            "VCpuInfo": {
+                "DefaultVCpus": 8,
+                "DefaultCores": 4,
+                "DefaultThreadsPerCore": 2,
+                "ValidCores": [
+                    2,
+                    4
+                ],
+                "ValidThreadsPerCore": [
+                    1,
+                    2
+                ]
+            },
+            "MemoryInfo": {
+                "SizeInMiB": 16384
+            },
+            "InstanceStorageSupported": false,
+            "InstanceStorageInfo": {
+                "EbsInfo": {
+                    "EbsStorageSupported": false,
+                    "EbsStorageInfo": {
+                        "VolumeTypes": [
+                            "standard"
+                        ],
+                        "VolumeSizeInGiBMin": 1,
+                        "VolumeSizeInGiBMax": 1024
+                    }
+                },
+                "InstanceStorageSupported": false,
+                "InstanceStorageInfo": {
+                    "VolumeTypes": [
+                        "standard"
+                    ],
+                    "VolumeSizeInGiBMin": 1,
+                    "VolumeSizeInGiBMax": 1024
+                }
+            },
+            "GpuInfo": {
+                "GPUsSupported": false,
+                "GPUSupported": false,
+                "GPUSupportedOnDemand": false,
+                "GPUSupportedSpot": false
+            },
+            "FpgaInfo": {
+                "FPGAsSupported": false,
+                "FPGASupported": false,
+                "FPGASupportedOnDemand": false,
+                "FPGASupportedSpot": false
+            },
+            "InferenceAcceleratorInfo": {
+                "InferenceAcceleratorsSupported": false,
+                "InferenceAcceleratorsSupportedOnDemand": false,
+                "InferenceAcceleratorsSupportedSpot": false
+            },
+            "EbsInfo": {
+                "EbsOptimizedSupport": "default",
+                "EncryptionSupport": "supported",
+                "EbsOptimizedInfo": {
+                    "BaselineBandwidthInMbps": 2500,
+                    "BaselineThroughputInMBps": 312.5,
+                    "BaselineIops": 12000,
+                    "MaximumBandwidthInMbps": 10000,
+                    "MaximumThroughputInMBps": 1250,
+                    "MaximumIops": 40000
+                },
+                "NvmeSupport": "required"
+            },
+            "NetworkInfo": {
+                "NetworkPerformance": "Up to 12.5 Gigabit",
+                "MaximumNetworkInterfaces": 4,
+                "MaximumNetworkCards": 1,
+                "DefaultNetworkCardIndex": 0,
+                "NetworkCards": [
+                    {
+                        "NetworkCardIndex": 0,
+                        "NetworkPerformance": "Up to 12.5 Gigabit",
+                        "MaximumNetworkInterfaces": 4
+                    }
+                ],
+                "Ipv4AddressesPerInterface": 15,
+                "Ipv6AddressesPerInterface": 15,
+                "Ipv6Supported": true,
+                "EnaSupport": "required",
+                "EfaSupported": false,
+                "EncryptionInTransitSupported": true
+            },
+            "PlacementGroupInfo": {
+                "SupportedStrategies": [
+                    "cluster",
+                    "partition",
+                    "spread"
+                ]
+            },
+            "HibernationSupported": false,
+            "BurstablePerformanceSupported": false,
+            "DedicatedHostsSupported": true,
+            "AutoRecoverySupported": true,
+            "SupportedBootModes": [
+                "legacy-bios",
+                "uefi"
+            ]
+        }
+    ]
+}
diff --git a/plugins/aws/test/resources/service_quotas_test.py b/plugins/aws/test/resources/service_quotas_test.py
@@ -6,7 +6,7 @@
 from fix_plugin_aws.resource.base import AwsResource
 from fix_plugin_aws.aws_client import AwsClient
 from fix_plugin_aws.resource.base import GraphBuilder, AwsRegion
-from fix_plugin_aws.resource.ec2 import AwsEc2InstanceType, AwsEc2Vpc
+from fix_plugin_aws.resource.ec2 import AwsEc2InstanceType, AwsEc2Instance, AwsEc2Vpc
 from fix_plugin_aws.resource.elbv2 import AwsAlb
 from fix_plugin_aws.resource.iam import AwsIamServerCertificate
 from fix_plugin_aws.resource.service_quotas import AwsServiceQuota, RegionalQuotas
@@ -20,11 +20,10 @@ def test_service_quotas() -> None:
 
 
 def test_instance_type_quotas() -> None:
-    _, builder = round_trip_for(AwsServiceQuota, "usage", "quota_type")
-    AwsEc2InstanceType.collect_resources(builder)
+    _, builder = round_trip_for(AwsServiceQuota, "usage", "quota_type", collect_also=[AwsEc2Instance])
     for _, it in builder.global_instance_types.items():
         builder.add_node(it, {})
-    expect_quotas(builder, 3)
+    expect_quotas(builder, 5)
 
 
 def test_volume_type_quotas() -> None:

Original file line number	Diff line number	Diff line change
`@@ -17,6 +17,7 @@ def test_instance_type(builder: GraphBuilder) -> None:`
`17`	`17`	`cloud_instance_data, ["aws", instance_type, "pricing", builder.region.id, "linux", "ondemand"]`
`18`	`18`	`)`
`19`	`19`	`eu_builder = builder.for_region(AwsRegion(id="eu-central-1"))`
	`20`	`+ builder.global_instance_types[instance_type] = AwsEc2InstanceType(id=instance_type)`
`20`	`21`	`m4l_eu: AwsEc2InstanceType = eu_builder.instance_type(eu_builder.region, instance_type) # type: ignore`
`21`	`22`	`assert m4l != m4l_eu`
`22`	`23`	`assert m4l_eu == eu_builder.instance_type(eu_builder.region, instance_type)`