[azure][feat] Monitoring and KeyVault resources (#2156)

Author: aquamatthias

fixlib/fixlib/json_bender.py

@@ -622,3 +622,7 @@ def bend_with_context(inner: Mapping, transport: Transport) -> Any:
 
     context = {} if context is None else context
     return bend_with_context(mapping, Transport(source, context))
+
+
+Lower = F(lambda s: s.lower() if isinstance(s, str) else s)
+Upper = F(lambda s: s.upper() if isinstance(s, str) else s)

plugins/azure/fix_plugin_azure/__init__.py

@@ -132,7 +132,7 @@ def list_all(resource: Type[T], config: AzureConfig, credentials: AzureCredentia
     if resource.api_spec is None:
         return []
     client = MicrosoftClient.create(config, credentials, "global")
-    return [resource.from_api(js) for js in client.list(resource.api_spec)]
+    return [rs for js in client.list(resource.api_spec) if (rs := resource.from_api(js))]
 
 
 def collect_account_proxy(collector_arg: AzureCollectorArg, queue: multiprocessing.Queue) -> None:  # type: ignore

plugins/azure/fix_plugin_azure/collector.py

@@ -28,13 +28,15 @@
     resources as graph_resources,
     MicrosoftGraphOrganizationRoot,
 )
+from fix_plugin_azure.resource.monitor import resources as monitor_resources
 from fix_plugin_azure.resource.network import (
     AzureExpressRoutePortsLocation,
     AzureNetworkVirtualApplianceSku,
     AzureNetworkUsage,
     resources as network_resources,
 )
 from fix_plugin_azure.resource.mysql import AzureMysqlServerType, resources as mysql_resources
+from fix_plugin_azure.resource.keyvault import resources as keyvault_resources
 from fix_plugin_azure.resource.sql_server import resources as sql_resources
 from fix_plugin_azure.resource.postgresql import (
     AzurePostgresqlServerType,
@@ -68,6 +70,8 @@ def resource_with_params(clazz: Type[MicrosoftResource], param: str) -> bool:
     + sql_resources
     + mysql_resources
     + postgresql_resources
+    + monitor_resources
+    + keyvault_resources
 )
 all_resources = subscription_resources + graph_resources  # defines all resource kinds. used in model check
 

plugins/azure/fix_plugin_azure/resource/base.py

@@ -52,7 +52,7 @@ def get_client(subscription_id: str) -> MicrosoftClient:
 
 
 def parse_json(
-    json: Json, clazz: Type[T], builder: GraphBuilder, mapping: Optional[Dict[str, Bender]] = None
+    json: Json, clazz: Type[T], builder: Optional[GraphBuilder] = None, mapping: Optional[Dict[str, Bender]] = None
 ) -> Optional[T]:
     """
     Use this method to parse json into a class. If the json can not be parsed, the error is reported to the core.
@@ -67,14 +67,19 @@ def parse_json(
         mapped = bend(mapping, json) if mapping is not None else json
         return from_json(mapped, clazz)
     except Exception as e:
-        # report and log the error
-        builder.core_feedback.error(f"Failed to parse json into {clazz.__name__}: {e}. Source: {json}", log)
-        # based on the strict flag, either raise the exception or return None
-        if builder.config.discard_account_on_resource_error:
-            raise
+        message = f"Failed to parse json into {clazz.__name__}: {e}. Source: {json}"
+        if builder:
+            # report and log the error
+            builder.core_feedback.error(message, log)
+            # based on the strict flag, either raise the exception or return None
+            if builder.config.discard_account_on_resource_error:
+                raise
+        else:
+            log.warning(message)
         return None
 
 
+@define(eq=False, slots=False)
 class MicrosoftResource(BaseResource):
     kind: ClassVar[str] = "microsoft_resource"
     # The mapping to transform the incoming API json into the internal representation.
@@ -224,19 +229,20 @@ def collect(
         result: List[MicrosoftResourceType] = []
         for js in raw:
             # map from api
-            instance = cls.from_api(js)
-            instance.pre_process(builder, js)
-            # add to graph
-            if (added := builder.add_node(instance, js)) is not None:
-                # post process
-                added.post_process(builder, js)
-                result.append(added)
+            if instance := cls.from_api(js, builder):
+                instance.pre_process(builder, js)
+                # add to graph
+                if (added := builder.add_node(instance, js)) is not None:
+                    # post process
+                    added.post_process(builder, js)
+                    result.append(added)
         return result
 
     @classmethod
-    def from_api(cls: Type[MicrosoftResourceType], json: Json) -> MicrosoftResourceType:
-        mapped = bend(cls.mapping, json)
-        return cls.from_json(mapped)
+    def from_api(
+        cls: Type[MicrosoftResourceType], json: Json, builder: Optional[GraphBuilder] = None
+    ) -> Optional[MicrosoftResourceType]:
+        return parse_json(json, cls, builder, cls.mapping)
 
     @classmethod
     def called_collect_apis(cls) -> List[MicrosoftRestSpec]:
@@ -504,21 +510,6 @@ class AzurePrincipalClient:
     principal_id: Optional[str] = field(default=None, metadata={'description': 'The principal id of user assigned identity.'})  # fmt: skip
 
 
-@define(eq=False, slots=False)
-class AzureManagedServiceIdentity:
-    kind: ClassVar[str] = "azure_managed_service_identity"
-    mapping: ClassVar[Dict[str, Bender]] = {
-        "principal_id": S("principalId"),
-        "tenant_id": S("tenantId"),
-        "type": S("type"),
-        "user_assigned_identities": S("userAssignedIdentities"),
-    }
-    principal_id: Optional[str] = field(default=None, metadata={'description': 'The principal id of the system assigned identity. This property will only be provided for a system assigned identity.'})  # fmt: skip
-    tenant_id: Optional[str] = field(default=None, metadata={'description': 'The tenant id of the system assigned identity. This property will only be provided for a system assigned identity.'})  # fmt: skip
-    type: Optional[str] = field(default=None, metadata={'description': 'The type of identity used for the resource. The type SystemAssigned, UserAssigned includes both an implicitly created identity and a set of user assigned identities. The type None will remove any identities from the virtual machine.'})  # fmt: skip
-    user_assigned_identities: Optional[Dict[str, AzurePrincipalClient]] = field(default=None, metadata={'description': 'The list of user identities associated with resource. The user identity dictionary key references will be ARM resource ids in the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} .'})  # fmt: skip
-
-
 @define(eq=False, slots=False)
 class AzurePrivateLinkServiceConnectionState:
     kind: ClassVar[str] = "azure_private_link_service_connection_state"
@@ -561,6 +552,7 @@ class AzureSubscription(MicrosoftResource, BaseAccount):
         "id": S("subscriptionId"),
         "tags": S("tags", default={}),
         "authorization_source": S("authorizationSource"),
+        "name": S("displayName"),
         "display_name": S("displayName"),
         "managed_by_tenants": S("managedByTenants", default=[]) >> ForallBend(S("tenantId")),
         "state": S("state"),
@@ -636,6 +628,23 @@ class AzureProxyResource:
     type: Optional[str] = field(default=None, metadata={'description': 'The type of the resource. E.g. Microsoft.Compute/virtualMachines or Microsoft.Storage/storageAccounts '})  # fmt: skip
 
 
+@define(eq=False, slots=False)
+class AzureIdentity:
+    kind: ClassVar[str] = "azure_identity"
+    mapping: ClassVar[Dict[str, Bender]] = {
+        "client_id": S("clientId"),
+        "principal_id": S("principalId"),
+        "tenant_id": S("tenantId"),
+        "type": S("type"),
+        "user_assigned_identities": S("userAssignedIdentities"),
+    }
+    client_id: Optional[str] = field(default=None, metadata={'description': 'The client id of user assigned identity.'})  # fmt: skip
+    principal_id: Optional[str] = field(default=None, metadata={'description': 'The principal ID of resource identity.'})  # fmt: skip
+    tenant_id: Optional[str] = field(default=None, metadata={"description": "The tenant ID of resource."})
+    type: Optional[str] = field(default=None, metadata={"description": "Type of managed service identity."})
+    user_assigned_identities: Optional[Dict[str, AzureUserAssignedIdentity]] = field(default=None, metadata={'description': 'The list of user identities associated with the resource. The user identity dictionary key references will be ARM resource ids in the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName} .'})  # fmt: skip
+
+
 @define(eq=False, slots=False)
 class AzureSku:
     kind: ClassVar[str] = "azure_sku"

plugins/azure/fix_plugin_azure/resource/containerservice.py

@@ -12,7 +12,7 @@
     AzureExtendedLocation,
     AzureUserAssignedIdentity,
     AzurePrincipalClient,
-    AzureManagedServiceIdentity,
+    AzureIdentity,
 )
 from fixlib.baseresources import BaseManagedKubernetesClusterProvider, BaseSnapshot, EdgeType, ModelReference
 from fixlib.json_bender import Bender, S, Bend, ForallBend
@@ -86,12 +86,12 @@ class AzureFleet(MicrosoftResource):
         "e_tag": S("eTag"),
         "resource_group": S("resourceGroup"),
         "hub_profile": S("properties", "hubProfile") >> Bend(AzureFleetHubProfile.mapping),
-        "azure_fleet_identity": S("identity") >> Bend(AzureManagedServiceIdentity.mapping),
+        "azure_fleet_identity": S("identity") >> Bend(AzureIdentity.mapping),
         "provisioning_state": S("properties", "provisioningState"),
     }
     e_tag: Optional[str] = field(default=None, metadata={'description': 'If eTag is provided in the response body, it may also be provided as a header per the normal etag convention. Entity tags are used for comparing two or more entities from the same requested resource. HTTP/1.1 uses entity tags in the etag (section 14.19), If-Match (section 14.24), If-None-Match (section 14.26), and If-Range (section 14.27) header fields.'})  # fmt: skip
     hub_profile: Optional[AzureFleetHubProfile] = field(default=None, metadata={'description': 'The FleetHubProfile configures the fleet hub.'})  # fmt: skip
-    azure_fleet_identity: Optional[AzureManagedServiceIdentity] = field(default=None, metadata={'description': 'Managed service identity (system assigned and/or user assigned identities)'})  # fmt: skip
+    azure_fleet_identity: Optional[AzureIdentity] = field(default=None, metadata={'description': 'Managed service identity (system assigned and/or user assigned identities)'})  # fmt: skip
     resource_group: Optional[str] = field(default=None, metadata={"description": "Resource group name"})
     _cluster_resource_ids: Optional[List[str]] = field(
         default=None, metadata={"description": "Reference to the cluster IDs"}