Skip to content

Commit 4fffcfa

Browse files
1101-1aquamatthias
1101-1
and
aquamatthias
authored
[azure][feat] Add cosmos-db resources collection (#2167)
Co-authored-by: Matthias Veit <matthias_veit@yahoo.de>
1 parent 4b9b718 commit 4fffcfa

19 files changed

+3028
-14
lines changed

‎fixcore/fixcore/report/report_config.py‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -75,14 +75,18 @@ def from_config(cfg: ConfigEntity) -> List[ReportCheck]:
7575
@staticmethod
7676
def from_json(js: Json) -> List[ReportCheck]:
7777
def report_check(pdr: str, svc: str, check: Json) -> ReportCheck:
78-
cr = check.copy()
79-
cr["provider"] = pdr
80-
cr["service"] = svc
81-
cr["id"] = f"{pdr}_{svc}_{check['name']}"
82-
# handle legacy result_kind
83-
if "result_kind" in cr and "result_kinds" not in cr:
84-
cr["result_kinds"] = [cr.pop("result_kind")]
85-
return from_js(cr, ReportCheck)
78+
try:
79+
cr = check.copy()
80+
cr["provider"] = pdr
81+
cr["service"] = svc
82+
cr["id"] = f"{pdr}_{svc}_{check['name']}"
83+
# handle legacy result_kind
84+
if "result_kind" in cr and "result_kinds" not in cr:
85+
cr["result_kinds"] = [cr.pop("result_kind")]
86+
return from_js(cr, ReportCheck)
87+
except Exception:
88+
log.error(f"Failed to load check {check}")
89+
raise
8690

8791
pdr = js["provider"]
8892
svc = js["service"]

‎fixcore/tests/fixcore/report/report_config_test.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
from typing import List
2+
3+
import pytest
4+
from fixcore.report import ReportCheck
5+
from fixcore.report.report_config import ReportCheckCollectionConfig
6+
from fixcore.types import Json
7+
from fixlib.json import to_json
8+
9+
10+
def test_report_config(inspection_checks: List[ReportCheck]) -> None:
11+
valid_js: List[Json] = to_json(inspection_checks) # type: ignore
12+
for a in valid_js:
13+
a["name"] = a.pop("id")
14+
icj = {"provider": "a", "service": "b", "checks": valid_js}
15+
ReportCheckCollectionConfig.from_json(icj)
16+
with pytest.raises(Exception):
17+
ReportCheckCollectionConfig.from_json({"provider": "a", "service": "b", "checks": [{}]})

‎plugins/azure/fix_plugin_azure/collector.py‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@
4343
)
4444
from fix_plugin_azure.resource.security import resources as security_resources
4545
from fix_plugin_azure.resource.sql_server import resources as sql_resources
46+
from fix_plugin_azure.resource.cosmosdb import (
47+
AzureCosmosDBLocation,
48+
resources as cosmosdb_resources,
49+
)
4650
from fix_plugin_azure.resource.storage import AzureStorageAccountUsage, AzureStorageSku, resources as storage_resources
4751
from fix_plugin_azure.resource.web import resources as web_resources
4852
from fixlib.baseresources import Cloud, GraphRoot, BaseAccount, BaseRegion
@@ -66,6 +70,7 @@ def resource_with_params(clazz: Type[MicrosoftResource], param: str) -> bool:
6670
+ aks_resources
6771
+ authorization_resources
6872
+ compute_resources
73+
+ cosmosdb_resources
6974
+ keyvault_resources
7075
+ monitor_resources
7176
+ mysql_resources
@@ -219,14 +224,17 @@ def collect_with(self, builder: GraphBuilder, locations: Dict[str, BaseRegion])
219224
def remove_unused(self) -> None:
220225
remove_nodes = []
221226

222-
def rm_nodes(cls, ignore_kinds: Optional[Type[Any]] = None) -> None: # type: ignore
227+
def rm_nodes(cls, ignore_kinds: Optional[Type[Any]] = None, check_pred: bool = True) -> None: # type: ignore
223228
for node in self.graph.nodes:
224229
if not isinstance(node, cls):
225230
continue
226-
pred = list(self.graph.predecessors(node))
231+
if check_pred:
232+
nodes = list(self.graph.predecessors(node))
233+
else:
234+
nodes = list(self.graph.successors(node))
227235
if ignore_kinds is not None:
228-
pred = [p for p in pred if not isinstance(p, ignore_kinds)]
229-
if not pred:
236+
nodes = [n for n in nodes if not isinstance(n, ignore_kinds)]
237+
if not nodes:
230238
remove_nodes.append(node)
231239
self._delete_nodes(remove_nodes)
232240
log.debug(f"Removing {len(remove_nodes)} unreferenced nodes of type {cls}")
@@ -249,6 +257,8 @@ def remove_usage_zero_value() -> None:
249257
rm_nodes(AzureStorageSku, AzureLocation)
250258
rm_nodes(AzureMysqlServerType, AzureSubscription)
251259
rm_nodes(AzurePostgresqlServerType, AzureSubscription)
260+
rm_nodes(AzureCosmosDBLocation, AzureSubscription, check_pred=False)
261+
rm_nodes(AzureLocation, check_pred=False)
252262
remove_usage_zero_value()
253263

254264
def after_collect(self) -> None:

‎plugins/azure/fix_plugin_azure/resource/cosmosdb.py‎

Lines changed: 2247 additions & 0 deletions
Large diffs are not rendered by default.

‎plugins/azure/test/collector_test.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,8 @@ def test_collect(
4949
config, Cloud(id="azure"), azure_subscription, credentials, core_feedback
5050
)
5151
subscription_collector.collect()
52-
assert len(subscription_collector.graph.nodes) == 463
53-
assert len(subscription_collector.graph.edges) == 701
52+
assert len(subscription_collector.graph.nodes) == 492
53+
assert len(subscription_collector.graph.edges) == 754
5454

5555
graph_collector = MicrosoftGraphOrganizationCollector(
5656
config, Cloud(id="azure"), MicrosoftGraphOrganization(id="test", name="test"), credentials, core_feedback

‎plugins/azure/test/cosmos-db_test.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
from conftest import roundtrip_check
2+
from fix_plugin_azure.resource.base import GraphBuilder
3+
from fix_plugin_azure.resource.cosmosdb import (
4+
AzureCosmosDBCassandraCluster,
5+
AzureCosmosDBAccount,
6+
AzureCosmosDBRestorableAccount,
7+
AzureCosmosDBLocation,
8+
AzureCosmosDBMongoDBCluster,
9+
)
10+
11+
12+
def test_cassandra_cluster(builder: GraphBuilder) -> None:
13+
collected = roundtrip_check(AzureCosmosDBCassandraCluster, builder)
14+
assert len(collected) == 1
15+
16+
17+
def test_cosmos_db_account(builder: GraphBuilder) -> None:
18+
collected = roundtrip_check(AzureCosmosDBAccount, builder)
19+
assert len(collected) == 1
20+
21+
22+
def test_restorable_cosmos_db_account(builder: GraphBuilder) -> None:
23+
collected = roundtrip_check(AzureCosmosDBRestorableAccount, builder)
24+
assert len(collected) == 2
25+
26+
27+
def test_cosmos_db_location(builder: GraphBuilder) -> None:
28+
collected = roundtrip_check(AzureCosmosDBLocation, builder)
29+
assert len(collected) == 2
30+
31+
32+
def test_mongo_db_cluster(builder: GraphBuilder) -> None:
33+
collected = roundtrip_check(AzureCosmosDBMongoDBCluster, builder)
34+
assert len(collected) == 1

‎plugins/azure/test/files/cosmos-db/cassandraClusters.json‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
{
2+
"value": [
3+
{
4+
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.DocumentDB/cassandraClusters",
5+
"name": "cassandra-prod",
6+
"type": "Microsoft.DocumentDB/cassandraClusters",
7+
"location": "West US",
8+
"tags": {},
9+
"properties": {
10+
"provisioningState": "Succeeded",
11+
"delegatedManagementSubnetId": "/subscriptions/536e130b-d7d6-4ac7-98a5-de20d69588d2/resourceGroups/customer-vnet-rg/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/management",
12+
"cassandraVersion": "3.11",
13+
"hoursBetweenBackups": 24,
14+
"authenticationMethod": "Cassandra",
15+
"externalSeedNodes": [
16+
{
17+
"ipAddress": "10.52.221.2"
18+
},
19+
{
20+
"ipAddress": "10.52.221.3"
21+
},
22+
{
23+
"ipAddress": "10.52.221.4"
24+
}
25+
],
26+
"clusterNameOverride": "ClusterNameIllegalForAzureResource",
27+
"seedNodes": [
28+
{
29+
"ipAddress": "10.52.221.2"
30+
},
31+
{
32+
"ipAddress": "10.52.221.3"
33+
},
34+
{
35+
"ipAddress": "10.52.221.4"
36+
},
37+
{
38+
"ipAddress": "192.168.12.2"
39+
},
40+
{
41+
"ipAddress": "192.168.12.3"
42+
},
43+
{
44+
"ipAddress": "192.168.12.4"
45+
}
46+
],
47+
"clientCertificates": [
48+
{
49+
"pem": "-----BEGIN CERTIFICATE-----\n...Base64 encoded certificate...\n-----END CERTIFICATE-----"
50+
}
51+
],
52+
"externalGossipCertificates": [
53+
{
54+
"pem": "-----BEGIN CERTIFICATE-----\n...Base64 encoded certificate...\n-----END CERTIFICATE-----"
55+
}
56+
],
57+
"gossipCertificates": [
58+
{
59+
"pem": "-----BEGIN CERTIFICATE-----\n...Base64 encoded certificate...\n-----END CERTIFICATE-----"
60+
}
61+
]
62+
}
63+
}
64+
]
65+
}

‎plugins/azure/test/files/cosmos-db/dataCenters.json‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"value": [
3+
{
4+
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/cassandra-prod-rg/providers/Microsoft.DocumentDB/cassandraClusters/cassandra-prod/dataCenters",
5+
"name": "dc1",
6+
"type": "Microsoft.DocumentDB/cassandraClusters/dataCenters",
7+
"properties": {
8+
"provisioningState": "Succeeded",
9+
"dataCenterLocation": "West US 2",
10+
"delegatedSubnetId": "/subscriptions/536e130b-d7d6-4ac7-98a5-de20d69588d2/resourceGroups/customer-vnet-rg/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/dc1",
11+
"nodeCount": 9,
12+
"seedNodes": [
13+
{
14+
"ipAddress": "192.168.12.2"
15+
},
16+
{
17+
"ipAddress": "192.168.12.3"
18+
},
19+
{
20+
"ipAddress": "192.168.12.4"
21+
}
22+
],
23+
"base64EncodedCassandraYamlFragment": "Y29tcGFjdGlvbl90aHJvdWdocHV0X21iX3Blcl9zZWM6IDMyCmNvbXBhY3Rpb25fbGFyZ2VfcGFydGl0aW9uX3dhcm5pbmdfdGhyZXNob2xkX21iOiAxMDA="
24+
}
25+
}
26+
]
27+
}

‎plugins/azure/test/files/cosmos-db/databaseAccounts.json‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
{
2+
"value": [
3+
{
4+
"id": "/subscriptions/subid/resourceGroups/rg1/providers/Microsoft.DocumentDB/databaseAccounts/ddb1",
5+
"name": "ddb1",
6+
"location": "West US",
7+
"type": "Microsoft.DocumentDB/databaseAccounts",
8+
"kind": "GlobalDocumentDB",
9+
"tags": {},
10+
"properties": {
11+
"provisioningState": "Succeeded",
12+
"documentEndpoint": "https://ddb1.documents.azure.com:443/",
13+
"ipRules": [],
14+
"databaseAccountOfferType": "Standard",
15+
"disableKeyBasedMetadataWriteAccess": false,
16+
"consistencyPolicy": {
17+
"defaultConsistencyLevel": "Session",
18+
"maxIntervalInSeconds": 5,
19+
"maxStalenessPrefix": 100
20+
},
21+
"writeLocations": [
22+
{
23+
"id": "ddb1-eastus",
24+
"locationName": "East US",
25+
"documentEndpoint": "https://ddb1-eastus.documents.azure.com:443/",
26+
"provisioningState": "Succeeded",
27+
"failoverPriority": 0
28+
}
29+
],
30+
"readLocations": [
31+
{
32+
"id": "ddb1-eastus",
33+
"locationName": "East US",
34+
"documentEndpoint": "https://ddb1-eastus.documents.azure.com:443/",
35+
"provisioningState": "Succeeded",
36+
"failoverPriority": 0
37+
}
38+
],
39+
"locations": [
40+
{
41+
"id": "ddb1-eastus",
42+
"locationName": "East US",
43+
"documentEndpoint": "https://ddb1-eastus.documents.azure.com:443/",
44+
"provisioningState": "Succeeded",
45+
"failoverPriority": 0
46+
}
47+
],
48+
"failoverPolicies": [
49+
{
50+
"id": "ddb1-eastus",
51+
"locationName": "East US",
52+
"failoverPriority": 0
53+
}
54+
],
55+
"privateEndpointConnections": [
56+
{
57+
"id": "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.DocumentDB/databaseAccounts/account1/privateEndpointConnections/pe1",
58+
"properties": {
59+
"privateEndpoint": {
60+
"id": "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/privateEndpoints/pe1"
61+
},
62+
"privateLinkServiceConnectionState": {
63+
"status": "Approved",
64+
"actionsRequired": "None"
65+
}
66+
}
67+
}
68+
],
69+
"cors": [],
70+
"defaultIdentity": "FirstPartyIdentity",
71+
"enableFreeTier": false,
72+
"apiProperties": {},
73+
"enableAnalyticalStorage": true,
74+
"enableBurstCapacity": true,
75+
"analyticalStorageConfiguration": {
76+
"schemaType": "WellDefined"
77+
},
78+
"instanceId": "d9b26648-2f53-4541-b3d8-3044f4f9810d",
79+
"createMode": "Default",
80+
"backupPolicy": {
81+
"type": "Periodic",
82+
"periodicModeProperties": {
83+
"backupIntervalInMinutes": 240,
84+
"backupRetentionIntervalInHours": 720,
85+
"backupStorageRedundancy": "Geo"
86+
}
87+
},
88+
"networkAclBypass": "None",
89+
"networkAclBypassResourceIds": [],
90+
"keysMetadata": {
91+
"primaryMasterKey": {
92+
"generationTime": "2022-02-25T20:30:11Z"
93+
},
94+
"secondaryMasterKey": {
95+
"generationTime": "2022-02-25T20:30:11Z"
96+
},
97+
"primaryReadonlyMasterKey": {
98+
"generationTime": "2022-02-25T20:30:11Z"
99+
},
100+
"secondaryReadonlyMasterKey": {
101+
"generationTime": "2022-02-25T20:30:11Z"
102+
}
103+
},
104+
"enablePartitionMerge": true,
105+
"minimalTlsVersion": "Tls"
106+
},
107+
"systemData": {
108+
"createdAt": "2021-03-12T22:05:09Z"
109+
},
110+
"identity": {
111+
"type": "SystemAssigned,UserAssigned",
112+
"principalId": "52f4fef3-3c3f-4ff3-b52e-b5c9eeb68656",
113+
"tenantId": "33e01921-4d64-4f8c-a055-5bdaffd5e33d",
114+
"userAssignedIdentities": {
115+
"/subscriptions/fa5fc227-a624-475e-b696-cdd604c735bc/resourceGroups/eu2cgroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id1": {
116+
"clientId": "fbe75b66-01c5-4f87-a220-233af3270436",
117+
"principalId": "33e01921-4d64-4f8c-a055-5bdaffd5e33d"
118+
}
119+
}
120+
}
121+
}
122+
]
123+
}

‎plugins/azure/test/files/cosmos-db/locations.json‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
{
2+
"value": [
3+
{
4+
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.DocumentDB/locations/westus",
5+
"type": "Microsoft.DocumentDB/locations",
6+
"name": "westus",
7+
"properties": {
8+
"supportsAvailabilityZone": false,
9+
"isResidencyRestricted": false,
10+
"backupStorageRedundancies": [
11+
"Local",
12+
"Geo"
13+
],
14+
"isSubscriptionRegionAccessAllowedForRegular": true,
15+
"isSubscriptionRegionAccessAllowedForAz": false,
16+
"status": "Online"
17+
}
18+
},
19+
{
20+
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.DocumentDB/locations/centralus",
21+
"type": "Microsoft.DocumentDB/locations",
22+
"name": "centralus",
23+
"properties": {
24+
"supportsAvailabilityZone": true,
25+
"isResidencyRestricted": false,
26+
"backupStorageRedundancies": [
27+
"Zone",
28+
"Geo"
29+
],
30+
"isSubscriptionRegionAccessAllowedForRegular": false,
31+
"isSubscriptionRegionAccessAllowedForAz": true,
32+
"status": "Online"
33+
}
34+
}
35+
]
36+
}

0 commit comments

Comments
 (0)