[azure][feat] Improve SQL Server (#2151)

Signed-off-by: Lukas Lösche <lukas@some.engineering>
Co-authored-by: Lukas Lösche <lukas@some.engineering>

Author: aquamatthias

plugins/azure/fix_plugin_azure/collector.py

@@ -34,8 +34,8 @@
     AzureNetworkUsage,
     resources as network_resources,
 )
-from fix_plugin_azure.resource.sql import resources as sql_resources
 from fix_plugin_azure.resource.mysql import AzureMysqlCapability, AzureMysqlServerType, resources as mysql_resources
+from fix_plugin_azure.resource.sql_server import resources as sql_resources
 from fix_plugin_azure.resource.storage import AzureStorageAccountUsage, AzureStorageSku, resources as storage_resources
 from fixlib.baseresources import Cloud, GraphRoot, BaseAccount, BaseRegion
 from fixlib.core.actions import CoreFeedback, ErrorAccumulator

plugins/azure/fix_plugin_azure/resource/base.py

@@ -23,6 +23,7 @@
 from fixlib.config import current_config
 from fixlib.core.actions import CoreFeedback
 from fixlib.graph import Graph, EdgeKey, NodeSelector, ByNodeId
+from fixlib.json import from_json
 from fixlib.json_bender import AsFloat, Bender, bend, S, ForallBend, Bend
 from fixlib.lock import RWLock
 from fixlib.threading import ExecutorQueue
@@ -50,6 +51,30 @@ def get_client(subscription_id: str) -> MicrosoftClient:
 T = TypeVar("T")
 
 
+def parse_json(
+    json: Json, clazz: Type[T], builder: GraphBuilder, mapping: Optional[Dict[str, Bender]] = None
+) -> Optional[T]:
+    """
+    Use this method to parse json into a class. If the json can not be parsed, the error is reported to the core.
+    Based on configuration, either the exception is raised or None is returned.
+    :param json: the json to parse.
+    :param clazz: the class to parse into.
+    :param builder: the graph builder.
+    :param mapping: the optional mapping to apply before parsing.
+    :return: The parsed object or None.
+    """
+    try:
+        mapped = bend(mapping, json) if mapping is not None else json
+        return from_json(mapped, clazz)
+    except Exception as e:
+        # report and log the error
+        builder.core_feedback.error(f"Failed to parse json into {clazz.__name__}: {e}. Source: {json}", log)
+        # based on the strict flag, either raise the exception or return None
+        if builder.config.discard_account_on_resource_error:
+            raise
+        return None
+
+
 class MicrosoftResource(BaseResource):
     kind: ClassVar[str] = "microsoft_resource"
     # The mapping to transform the incoming API json into the internal representation.

plugins/azure/fix_plugin_azure/resource/sql.py renamed to plugins/azure/fix_plugin_azure/resource/sql_server.py

@@ -50,7 +50,7 @@ def test_collect(
     )
     subscription_collector.collect()
     assert len(subscription_collector.graph.nodes) == 407
-    assert len(subscription_collector.graph.edges) == 625
+    assert len(subscription_collector.graph.edges) == 627
 
     graph_collector = MicrosoftGraphOrganizationCollector(
         config, Cloud(id="azure"), MicrosoftGraphOrganization(id="test", name="test"), credentials, core_feedback

plugins/azure/test/collector_test.py

@@ -36,7 +36,7 @@ def list(self, spec: MicrosoftRestSpec, **kwargs: Any) -> List[Json]:
                 js = json.load(f)
 
                 if spec.expect_array:
-                    js = js[spec.access_path]
+                    js = js[spec.access_path] if spec.access_path else [js]
                 if spec.expect_array and isinstance(js, list):
                     return js
                 else:

plugins/azure/test/conftest.py

@@ -0,0 +1,20 @@
+{
+    "value": [
+        {
+            "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/foo/providers/Microsoft.Sql/servers/test-server111/extendedAuditingSettings/Default",
+            "name": "Default",
+            "properties": {
+                "auditActionsAndGroups": [],
+                "isAzureMonitorTargetEnabled": false,
+                "isManagedIdentityInUse": false,
+                "isStorageSecondaryKeyInUse": false,
+                "predicateExpression": "",
+                "retentionDays": 0,
+                "state": "Disabled",
+                "storageAccountSubscriptionId": "00000000-0000-0000-0000-000000000000",
+                "storageEndpoint": ""
+            },
+            "type": "Microsoft.Sql/servers/extendedAuditingSettings"
+        }
+    ]
+}

plugins/azure/test/files/sql/auditingSettings.json

@@ -0,0 +1,8 @@
+{
+    "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/foo/providers/Microsoft.Sql/servers/test-server111/databases/master/transparentDataEncryption/Current",
+    "name": "Current",
+    "properties": {
+        "state": "Disabled"
+    },
+    "type": "Microsoft.Sql/servers/databases/transparentDataEncryption"
+}

plugins/azure/test/files/sql/current.json

@@ -0,0 +1,15 @@
+{
+    "value": [
+        {
+            "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/foo/providers/Microsoft.Sql/servers/test-server111/encryptionProtector/current",
+            "kind": "servicemanaged",
+            "name": "current",
+            "properties": {
+                "autoRotationEnabled": false,
+                "serverKeyName": "ServiceManaged",
+                "serverKeyType": "ServiceManaged"
+            },
+            "type": "Microsoft.Sql/servers/encryptionProtector"
+        }
+    ]
+}

plugins/azure/test/files/sql/encryptionProtector.json

@@ -1,6 +1,6 @@
 from conftest import roundtrip_check
 from fix_plugin_azure.resource.base import GraphBuilder
-from fix_plugin_azure.resource.sql import (
+from fix_plugin_azure.resource.sql_server import (
     AzureSqlServerManagedInstancePool,
     AzureSqlServerManagedInstance,
     AzureSqlServer,
@@ -20,7 +20,10 @@ def test_sql_managed_instance(builder: GraphBuilder) -> None:
 
 def test_sql_server(builder: GraphBuilder) -> None:
     collected = roundtrip_check(AzureSqlServer, builder)
+    builder.executor.wait_for_submitted_work()
     assert len(collected) == 1
+    assert collected[0].blob_auditing_policy is not None
+    assert collected[0].encryption_protector is not None
 
 
 def test_sql_virtual_cluster(builder: GraphBuilder) -> None:

plugins/azure/test/sql_test.py renamed to plugins/azure/test/sql_server_test.py

@@ -389,6 +389,9 @@ def parse_spec(service: str, version: str, resolved: Json, file: Path) -> Iterat
                     "subscriptionId",
                     "location",
                     "scope",
+                    # "databaseName",
+                    # "serverName",
+                    # "resourceGroupName",
                 }
                 if len(param_names) == 0:
                     schema = method["responses"]["200"]["schema"]
@@ -420,9 +423,8 @@ def parse_spec(service: str, version: str, resolved: Json, file: Path) -> Iterat
                     )
                     if ssh := simple_shape(type_definition):
                         yield AzureRestSpec(ssh, info, type_definition, file)
-                    else:
-                        name = type_definition["ref"]  # name is added in the ref parser
-                        yield AzureRestSpec(name, info, type_definition, file)
+                    elif tdef := type_definition.get("ref"):
+                        yield AzureRestSpec(tdef, info, type_definition, file)
 
 
 class AzureModel:
@@ -617,7 +619,7 @@ def safe_idx(seq, index):
         "Checkout https://github.com/Azure/azure-rest-api-specs and set path in env"
     )
     model = AzureModel(Path(specs_path))
-    shapes = {spec.name: spec for spec in sorted(model.list_specs({"security"}), key=lambda x: x.name)}
+    shapes = {spec.name: spec for spec in sorted(model.list_specs({"sql"}), key=lambda x: x.name)}
     models = classes_from_model(shapes)
     for model in models.values():
         if model.name != "Resource":