[azure][feat] Add policies (#2141)

Author: aquamatthias

fixlib/fixlib/json_bender.py

@@ -368,20 +368,21 @@ def execute(self, source: Any) -> Any:
             return bool(source)
 
 
+def reformat_keys_to_snake(js: JsonElement) -> JsonElement:
+    if isinstance(js, dict):
+        return {snakecase(k): reformat_keys_to_snake(v) for k, v in js.items()}
+    elif isinstance(js, list):
+        return [reformat_keys_to_snake(v) for v in js]
+    else:
+        return js
+
+
 class ParseJson(Bender):
     def __init__(self, keys_to_snake: bool = False, **kwargs: Any):
         super().__init__(**kwargs)
         self._keys_to_snake = keys_to_snake
 
     def execute(self, source: Any) -> Any:
-        def reformat_keys_to_snake(js: JsonElement) -> JsonElement:
-            if isinstance(js, dict):
-                return {snakecase(k): reformat_keys_to_snake(v) for k, v in js.items()}
-            elif isinstance(js, list):
-                return [reformat_keys_to_snake(v) for v in js]
-            else:
-                return js
-
         if isinstance(source, str):
             try:
                 result = json.loads(source)

plugins/azure/fix_plugin_azure/collector.py

@@ -73,8 +73,6 @@ def __init__(
         account: BaseAccount,
         credentials: AzureCredentials,
         core_feedback: CoreFeedback,
-        global_resources: List[Type[MicrosoftResource]],
-        regional_resources: List[Type[MicrosoftResource]],
         task_data: Optional[Json] = None,
         max_resources_per_account: Optional[int] = None,
     ):
@@ -83,15 +81,9 @@ def __init__(
         self.account = account
         self.credentials = credentials
         self.core_feedback = core_feedback
-        self.global_resources = global_resources
-        self.regional_resources = regional_resources
         self.graph = Graph(root=account, max_nodes=max_resources_per_account)
         self.task_data = task_data
 
-    @abstractmethod
-    def locations(self, builder: GraphBuilder) -> Dict[str, BaseRegion]:
-        pass
-
     def collect(self) -> None:
         with ThreadPoolExecutor(
             thread_name_prefix=f"azure_{self.account.id}",
@@ -127,16 +119,15 @@ def get_last_run() -> Optional[datetime]:
                 config=self.config,
                 last_run_started_at=last_run,
             )
+
             # collect all locations
             locations = self.locations(builder)
             builder.location_lookup = locations
-            # collect all global resources
-            self.collect_resource_list("subscription", builder, self.global_resources)
-            # collect all regional resources
-            for location in locations.values():
-                self.collect_resource_list(location.safe_name, builder.with_location(location), self.regional_resources)
-            # wait for all work to finish
+
+            # collect all resources
+            self.collect_with(builder, locations)
             queue.wait_for_submitted_work()
+
             # connect nodes
             log.info(f"[Azure:{self.account.safe_name}] Connect resources and create edges.")
             for node, data in list(self.graph.nodes(data=True)):
@@ -146,18 +137,16 @@ def get_last_run() -> Optional[datetime]:
                     pass
                 else:
                     raise Exception(f"Only Azure resources expected, but got {node}")
-            # wait for all work to finish
             queue.wait_for_submitted_work()
-            # filter nodes
-            self.filter_nodes()
 
-            # post process nodes
+            # post-process nodes
+            self.remove_unused()
             for node, data in list(self.graph.nodes(data=True)):
                 if isinstance(node, MicrosoftResource):
                     node.after_collect(builder, data.get("source", {}))
 
             # delete unnecessary nodes after all work is completed
-            self.after_collect_filter()
+            self.after_collect()
             # report all accumulated errors
             error_accumulator.report_all(self.core_feedback)
             self.core_feedback.progress_done(self.account.id, 1, 1, context=[self.cloud.id])
@@ -183,7 +172,37 @@ def work_done(_: Future[None]) -> None:
         all_done.add_done_callback(work_done)
         return all_done
 
-    def filter_nodes(self) -> None:
+    @abstractmethod
+    def collect_with(self, builder: GraphBuilder, locations: Dict[str, BaseRegion]) -> None:
+        pass
+
+    @abstractmethod
+    def locations(self, builder: GraphBuilder) -> Dict[str, BaseRegion]:
+        pass
+
+    def remove_unused(self) -> None:
+        pass
+
+    def after_collect(self) -> None:
+        pass
+
+
+class AzureSubscriptionCollector(MicrosoftBaseCollector):
+    def locations(self, builder: GraphBuilder) -> Dict[str, BaseRegion]:
+        locations = AzureLocation.collect_resources(builder)
+        return CaseInsensitiveDict({loc.safe_name: loc for loc in locations})  # type: ignore
+
+    def collect_with(self, builder: GraphBuilder, locations: Dict[str, BaseRegion]) -> None:
+        # add deferred edge to organization
+        builder.submit_work("azure_all", MicrosoftGraphOrganization.deferred_edge_to_subscription, builder)
+        # collect all global and regional resources
+        regional_resources = [r for r in subscription_resources if resource_with_params(r, "location")]
+        global_resources = list(set(subscription_resources) - set(regional_resources))
+        self.collect_resource_list("subscription", builder, global_resources)
+        for location in locations.values():
+            self.collect_resource_list(location.safe_name, builder.with_location(location), regional_resources)
+
+    def remove_unused(self) -> None:
         remove_nodes = []
 
         def rm_nodes(cls, ignore_kinds: Optional[Type[Any]] = None) -> None:  # type: ignore
@@ -216,7 +235,7 @@ def remove_usage_zero_value() -> None:
         rm_nodes(AzureStorageSku, AzureLocation)
         remove_usage_zero_value()
 
-    def after_collect_filter(self) -> None:
+    def after_collect(self) -> None:
         # Filter unnecessary nodes such as AzureDiskTypePricing
         nodes_to_remove = []
         node_types = (AzureDiskTypePricing,)
@@ -227,70 +246,23 @@ def after_collect_filter(self) -> None:
             nodes_to_remove.append(node)
         self._delete_nodes(nodes_to_remove)
 
-    def _delete_nodes(self, nodes_to_delte: Any) -> None:
+    def _delete_nodes(self, nodes_to_delete: Any) -> None:
         removed = set()
-        for node in nodes_to_delte:
+        for node in nodes_to_delete:
             if node in removed:
                 continue
             removed.add(node)
             self.graph.remove_node(node)
-        nodes_to_delte.clear()
-
-
-class AzureSubscriptionCollector(MicrosoftBaseCollector):
-    def __init__(
-        self,
-        config: AzureConfig,
-        cloud: Cloud,
-        subscription: AzureSubscription,
-        credentials: AzureCredentials,
-        core_feedback: CoreFeedback,
-        task_data: Optional[Json] = None,
-        max_resources_per_account: Optional[int] = None,
-    ):
-        regional_resources = [r for r in subscription_resources if resource_with_params(r, "location")]
-        global_resources = list(set(subscription_resources) - set(regional_resources))
-        super().__init__(
-            config,
-            cloud,
-            subscription,
-            credentials,
-            core_feedback,
-            global_resources,
-            regional_resources,
-            task_data=task_data,
-            max_resources_per_account=max_resources_per_account,
-        )
-
-    def locations(self, builder: GraphBuilder) -> Dict[str, BaseRegion]:
-        locations = AzureLocation.collect_resources(builder)
-        return CaseInsensitiveDict({loc.safe_name: loc for loc in locations})  # type: ignore
+        nodes_to_delete.clear()
 
 
 class MicrosoftGraphOrganizationCollector(MicrosoftBaseCollector):
-    def __init__(
-        self,
-        config: AzureConfig,
-        cloud: Cloud,
-        organization: MicrosoftGraphOrganization,
-        credentials: AzureCredentials,
-        core_feedback: CoreFeedback,
-        task_data: Optional[Json] = None,
-        max_resources_per_account: Optional[int] = None,
-    ):
-        super().__init__(
-            config,
-            cloud,
-            organization,
-            credentials,
-            core_feedback,
-            [],
-            graph_resources,  # treat all resources as regional resources, attached to the organization root
-            task_data=task_data,
-            max_resources_per_account=max_resources_per_account,
-        )
 
     def locations(self, builder: GraphBuilder) -> Dict[str, BaseRegion]:
         root = MicrosoftGraphOrganizationRoot(id="organization_root")
         builder.add_node(root)
         return {"organization_root": root}
+
+    def collect_with(self, builder: GraphBuilder, locations: Dict[str, BaseRegion]) -> None:
+        for location in locations.values():  # all resources underneath the organization root
+            self.collect_resource_list(location.safe_name, builder.with_location(location), graph_resources)

plugins/azure/fix_plugin_azure/resource/microsoft_graph.py

@@ -1,16 +1,21 @@
 from __future__ import annotations
 
+import logging
 from datetime import datetime
-from typing import ClassVar, Dict, Optional, List, Type
+from typing import ClassVar, Dict, Optional, List, Type, Any
 
 from attr import define, field
+from isodate import parse_datetime
 
 from fix_plugin_azure.azure_client import RestApiSpec, MicrosoftRestSpec
 from fix_plugin_azure.resource.base import GraphBuilder, MicrosoftResource
 from fixlib.baseresources import BaseGroup, BaseRole, BaseAccount, BaseRegion, ModelReference, BaseUser
-from fixlib.json_bender import Bender, S, ForallBend, Bend, F, MapDict
+from fixlib.graph import BySearchCriteria, ByNodeId
+from fixlib.json_bender import Bender, S, ForallBend, Bend, F, MapDict, reformat_keys_to_snake
 from fixlib.types import Json
 
+log = logging.getLogger("fix.plugins.azure")
+
 
 @define(eq=False, slots=False)
 class MicrosoftGraphEntity(MicrosoftResource):
@@ -1097,12 +1102,97 @@ class MicrosoftGraphOrganization(MicrosoftGraphEntity, BaseAccount):
     tenant_type: Optional[str] = field(default=None, metadata={'description': 'Not nullable. Can be one of the following types: AAD - An enterprise identity access management (IAM) service that serves business-to-employee and business-to-business (B2B) scenarios. AAD B2C An identity access management (IAM) service that serves business-to-consumer (B2C) scenarios. CIAM - A customer identity & access management (CIAM) solution that provides an integrated platform to serve consumers, partners, and citizen scenarios.'})  # fmt: skip
     verified_domains: Optional[List[MicrosoftGraphVerifiedDomain]] = field(default=None, metadata={'description': 'The collection of domains associated with this tenant. Not nullable.'})  # fmt: skip
 
+    @classmethod
+    def deferred_edge_to_subscription(cls, builder: GraphBuilder) -> None:
+        for js in builder.client.list(cls.api_spec):
+            org = cls.from_api(js)
+            builder.add_deferred_edge(
+                BySearchCriteria(f'is({cls.kind}) and reported.id=="{org.id}"'), ByNodeId(builder.account.chksum)
+            )
+
 
 @define(eq=False, slots=False)
 class MicrosoftGraphOrganizationRoot(MicrosoftGraphEntity, BaseRegion):
     kind: ClassVar[str] = "microsoft_graph_organization_root"
 
 
+@define(eq=False, slots=False)
+class MicrosoftGraphPolicy(MicrosoftGraphEntity):
+    kind: ClassVar[str] = "microsoft_graph_policy"
+
+    policy_kind: Optional[str] = field(default=None, metadata={"description": "The kind of policy."})
+    enabled: Optional[bool] = field(default=None, metadata={"description": "Indicates whether the policy is enabled."})
+    description: Optional[str] = field(default=None, metadata={"description": "Description of the policy."})
+    policy: Optional[Json] = field(default=None, metadata={"description": "The policy."})
+
+    @classmethod
+    def collect_resources(cls, builder: GraphBuilder, **kwargs: Any) -> List[MicrosoftGraphPolicy]:
+        base = "https://graph.microsoft.com/v1.0/policies"
+        policies = {
+            "admin_consent request": RestApiSpec("graph", f"{base}/adminConsentRequestPolicy"),
+            "authorization": RestApiSpec("graph", f"{base}/authorizationPolicy"),
+            "authentication_flow": RestApiSpec("graph", f"{base}/authenticationFlowsPolicy"),
+            "authentication_method": RestApiSpec("graph", f"{base}/authenticationMethodsPolicy"),
+            "cross_tenant_access": RestApiSpec("graph", f"{base}/crossTenantAccessPolicy"),
+            "default_app_management": RestApiSpec("graph", f"{base}/defaultAppManagementPolicy"),
+            "device_registration": RestApiSpec("graph", f"{base}/deviceRegistrationPolicy"),
+            "identity_security_defaults_enforcement": RestApiSpec(
+                "graph", f"{base}/identitySecurityDefaultsEnforcementPolicy"
+            ),
+            "activity_based_timeout": RestApiSpec(
+                "graph", f"{base}/activityBasedTimeoutPolicies", expect_array=True, access_path="value"
+            ),
+            "app_management": RestApiSpec(
+                "graph", f"{base}/appManagementPolicies", expect_array=True, access_path="value"
+            ),
+            "authentication_strength": RestApiSpec(
+                "graph", f"{base}/authenticationStrengthPolicies", expect_array=True, access_path="value"
+            ),
+            "claims_mapping": RestApiSpec(
+                "graph", f"{base}/claimsMappingPolicies", expect_array=True, access_path="value"
+            ),
+            "conditional_access": RestApiSpec(
+                "graph", f"{base}/conditionalAccessPolicies", expect_array=True, access_path="value"
+            ),
+            "feature_rollout": RestApiSpec(
+                "graph", f"{base}/featureRolloutPolicies", expect_array=True, access_path="value"
+            ),
+            "home_realm_discovery": RestApiSpec(
+                "graph", f"{base}/homeRealmDiscoveryPolicies", expect_array=True, access_path="value"
+            ),
+            "token_issuance": RestApiSpec(
+                "graph", f"{base}/tokenIssuancePolicies", expect_array=True, access_path="value"
+            ),
+        }
+        result = []
+        for policy_kind, spec in policies.items():
+            try:
+                for response in builder.client.list(spec, **kwargs):
+                    rid = response.pop("id", policy_kind)
+                    name = response.pop("displayName", policy_kind)
+                    description = response.pop("description", None)
+                    enabled = response.pop("isEnabled", None) or response.pop("state", None) == "enabled" or True
+                    created = response.pop("createdDateTime", None)
+                    updated = response.pop("modifiedDateTime", None)
+                    policy = reformat_keys_to_snake({k: v for k, v in response.items() if not k.startswith("@odata")})
+                    gp = MicrosoftGraphPolicy(
+                        id=rid,
+                        policy_kind=policy_kind,
+                        name=name,
+                        ctime=parse_datetime(created) if created else None,
+                        mtime=parse_datetime(updated) if updated else None,
+                        description=description,
+                        policy=policy,  # type: ignore
+                        enabled=enabled,
+                    )
+                    builder.add_node(gp)
+                    result.append(gp)
+
+            except Exception as e:
+                log.warning(f"Error while collecting policies with service {spec.service}: {e}")
+        return result
+
+
 KindLookup = {
     "#microsoft.graph.user": MicrosoftGraphUser,
     "#microsoft.graph.group": MicrosoftGraphGroup,
@@ -1117,6 +1207,7 @@ class MicrosoftGraphOrganizationRoot(MicrosoftGraphEntity, BaseRegion):
 
 
 resources: List[Type[MicrosoftResource]] = [
+    MicrosoftGraphPolicy,
     MicrosoftGraphDevice,
     MicrosoftGraphServicePrincipal,
     MicrosoftGraphGroup,

plugins/azure/fix_plugin_azure/resource/security.py

@@ -5,7 +5,7 @@
 
 from fix_plugin_azure.azure_client import AzureResourceSpec
 from fix_plugin_azure.resource.base import MicrosoftResource, AzureSystemData, GraphBuilder
-from fixlib.json_bender import Bender, S, Bend, ForallBend
+from fixlib.json_bender import Bender, S, Bend, ForallBend, F
 from fixlib.types import Json
 
 
@@ -170,7 +170,28 @@ class AzureSecuritySetting(MicrosoftResource):
     enabled: Optional[bool] = field(default=None, metadata={"description": "Indicates whether the setting is enabled."})
 
 
+@define(eq=False, slots=False)
+class AzureAutoProvisioningSetting(MicrosoftResource):
+    kind: ClassVar[str] = "azure_auto_provisioning_setting"
+    api_spec: ClassVar[AzureResourceSpec] = AzureResourceSpec(
+        service="security",
+        version="2017-08-01-preview",
+        path="/subscriptions/{subscriptionId}/providers/Microsoft.Security/autoProvisioningSettings",
+        path_parameters=["subscriptionId"],
+        query_parameters=["api-version"],
+        access_path="value",
+        expect_array=True,
+    )
+    mapping: ClassVar[Dict[str, Bender]] = {
+        "id": S("id"),
+        "name": S("name"),
+        "auto_provision": S("properties", "autoProvision") >> F(lambda x: x == "On"),
+    }
+    auto_provision: Optional[bool] = field(default=None, metadata={'description': 'describes properties of an auto provisioning setting'})  # fmt: skip
+
+
 resources: List[Type[MicrosoftResource]] = [
+    AzureAutoProvisioningSetting,
     AzureSecurityAssessment,
     AzureSecurityPricing,
     AzureSecurityServerVulnerabilityAssessmentsSetting,