Skip to content

Commit 6a68104

Browse files
1101-11101-1
authored
[gcp][feat] Add filestore service collection (#2277)
1 parent 6d185bb commit 6a68104

File tree

7 files changed

+502
-4
lines changed

7 files changed

+502
-4
lines changed

‎plugins/gcp/fix_plugin_gcp/collector.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
from fix_plugin_gcp.config import GcpConfig
66
from fix_plugin_gcp.gcp_client import GcpApiSpec
7-
from fix_plugin_gcp.resources import compute, container, billing, sqladmin, storage, aiplatform, firestore
7+
from fix_plugin_gcp.resources import compute, container, billing, sqladmin, storage, aiplatform, firestore, filestore
88
from fix_plugin_gcp.resources.base import GcpResource, GcpProject, ExecutorQueue, GraphBuilder, GcpRegion, GcpZone
99
from fix_plugin_gcp.utils import Credentials
1010
from fixlib.baseresources import Cloud
@@ -20,6 +20,7 @@
2020
+ storage.resources
2121
+ aiplatform.resources
2222
+ firestore.resources
23+
+ filestore.resources
2324
)
2425

2526

‎plugins/gcp/fix_plugin_gcp/resources/filestore.py‎

Lines changed: 327 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,327 @@
1+
from datetime import datetime
2+
import logging
3+
from typing import ClassVar, Dict, Optional, List, Type, Any
4+
5+
from attr import define, field
6+
7+
from fix_plugin_gcp.gcp_client import GcpApiSpec
8+
from fix_plugin_gcp.resources.base import GraphBuilder, GcpErrorHandler, GcpResource, GcpDeprecationStatus
9+
from fixlib.baseresources import BaseNetworkShare, ModelReference
10+
from fixlib.json_bender import Bender, S, Bend, ForallBend
11+
from fixlib.types import Json
12+
13+
log = logging.getLogger("fix.plugins.gcp")
14+
15+
16+
service_name = "filestore"
17+
18+
19+
@define(eq=False, slots=False)
20+
class GcpFilestoreBackup(GcpResource):
21+
kind: ClassVar[str] = "gcp_filestore_backup"
22+
_kind_display: ClassVar[str] = "GCP Filestore Backup"
23+
_kind_description: ClassVar[str] = (
24+
"GCP Filestore Backup is a service that allows you to create backups of your Filestore instances."
25+
" It provides a way to protect your data and restore it in case of data loss."
26+
)
27+
_docs_url: ClassVar[str] = "https://cloud.google.com/filestore/docs/backups"
28+
_kind_service: ClassVar[Optional[str]] = "filestore"
29+
_metadata: ClassVar[Dict[str, Any]] = {"icon": "backup", "group": "storage"}
30+
api_spec: ClassVar[GcpApiSpec] = GcpApiSpec(
31+
service="file",
32+
version="v1",
33+
accessors=["projects", "locations", "backups"],
34+
action="list",
35+
request_parameter={"parent": "projects/{project}/locations/-"},
36+
request_parameter_in={"project"},
37+
response_path="backups",
38+
response_regional_sub_path=None,
39+
)
40+
mapping: ClassVar[Dict[str, Bender]] = {
41+
"id": S("name").or_else(S("id")).or_else(S("selfLink")),
42+
"name": S("name"),
43+
"ctime": S("creationTimestamp"),
44+
"description": S("description"),
45+
"link": S("selfLink"),
46+
"label_fingerprint": S("labelFingerprint"),
47+
"deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping),
48+
"capacity_gb": S("capacityGb"),
49+
"create_time": S("createTime"),
50+
"download_bytes": S("downloadBytes"),
51+
"file_system_protocol": S("fileSystemProtocol"),
52+
"kms_key": S("kmsKey"),
53+
"satisfies_pzi": S("satisfiesPzi"),
54+
"satisfies_pzs": S("satisfiesPzs"),
55+
"source_file_share": S("sourceFileShare"),
56+
"source_instance": S("sourceInstance"),
57+
"source_instance_tier": S("sourceInstanceTier"),
58+
"state": S("state"),
59+
"storage_bytes": S("storageBytes"),
60+
"tags": S("tags", default={}),
61+
}
62+
capacity_gb: Optional[str] = field(default=None)
63+
create_time: Optional[datetime] = field(default=None)
64+
download_bytes: Optional[str] = field(default=None)
65+
file_system_protocol: Optional[str] = field(default=None)
66+
kms_key: Optional[str] = field(default=None)
67+
satisfies_pzi: Optional[bool] = field(default=None)
68+
satisfies_pzs: Optional[bool] = field(default=None)
69+
source_file_share: Optional[str] = field(default=None)
70+
source_instance: Optional[str] = field(default=None)
71+
source_instance_tier: Optional[str] = field(default=None)
72+
state: Optional[str] = field(default=None)
73+
storage_bytes: Optional[str] = field(default=None)
74+
75+
76+
@define(eq=False, slots=False)
77+
class GcpNfsExportOptions:
78+
kind: ClassVar[str] = "gcp_nfs_export_options"
79+
mapping: ClassVar[Dict[str, Bender]] = {
80+
"access_mode": S("accessMode"),
81+
"anon_gid": S("anonGid"),
82+
"anon_uid": S("anonUid"),
83+
"ip_ranges": S("ipRanges", default=[]),
84+
"squash_mode": S("squashMode"),
85+
}
86+
access_mode: Optional[str] = field(default=None)
87+
anon_gid: Optional[str] = field(default=None)
88+
anon_uid: Optional[str] = field(default=None)
89+
ip_ranges: Optional[List[str]] = field(default=None)
90+
squash_mode: Optional[str] = field(default=None)
91+
92+
93+
@define(eq=False, slots=False)
94+
class GcpFileShareConfig:
95+
kind: ClassVar[str] = "gcp_file_share_config"
96+
mapping: ClassVar[Dict[str, Bender]] = {
97+
"capacity_gb": S("capacityGb"),
98+
"name": S("name"),
99+
"nfs_export_options": S("nfsExportOptions", default=[]) >> ForallBend(GcpNfsExportOptions.mapping),
100+
"source_backup": S("sourceBackup"),
101+
}
102+
capacity_gb: Optional[str] = field(default=None)
103+
name: Optional[str] = field(default=None)
104+
nfs_export_options: Optional[List[GcpNfsExportOptions]] = field(default=None)
105+
source_backup: Optional[str] = field(default=None)
106+
107+
108+
@define(eq=False, slots=False)
109+
class GcpNetworkConfig:
110+
kind: ClassVar[str] = "gcp_network_config"
111+
mapping: ClassVar[Dict[str, Bender]] = {
112+
"connect_mode": S("connectMode"),
113+
"ip_addresses": S("ipAddresses", default=[]),
114+
"modes": S("modes", default=[]),
115+
"network": S("network"),
116+
"reserved_ip_range": S("reservedIpRange"),
117+
}
118+
connect_mode: Optional[str] = field(default=None)
119+
ip_addresses: Optional[List[str]] = field(default=None)
120+
modes: Optional[List[str]] = field(default=None)
121+
network: Optional[str] = field(default=None)
122+
reserved_ip_range: Optional[str] = field(default=None)
123+
124+
125+
@define(eq=False, slots=False)
126+
class GcpPerformanceConfig:
127+
kind: ClassVar[str] = "gcp_performance_config"
128+
mapping: ClassVar[Dict[str, Bender]] = {
129+
"fixed_iops": S("fixedIops", "maxReadIops"),
130+
"iops_per_tb": S("iopsPerTb", "maxReadIopsPerTb"),
131+
}
132+
fixed_iops: Optional[str] = field(default=None)
133+
iops_per_tb: Optional[str] = field(default=None)
134+
135+
136+
@define(eq=False, slots=False)
137+
class GcpPerformanceLimits:
138+
kind: ClassVar[str] = "gcp_performance_limits"
139+
mapping: ClassVar[Dict[str, Bender]] = {
140+
"max_read_iops": S("maxReadIops"),
141+
"max_read_throughput_bps": S("maxReadThroughputBps"),
142+
"max_write_iops": S("maxWriteIops"),
143+
"max_write_throughput_bps": S("maxWriteThroughputBps"),
144+
}
145+
max_read_iops: Optional[str] = field(default=None)
146+
max_read_throughput_bps: Optional[str] = field(default=None)
147+
max_write_iops: Optional[str] = field(default=None)
148+
max_write_throughput_bps: Optional[str] = field(default=None)
149+
150+
151+
@define(eq=False, slots=False)
152+
class GcpReplicaConfig:
153+
kind: ClassVar[str] = "gcp_replica_config"
154+
mapping: ClassVar[Dict[str, Bender]] = {
155+
"last_active_sync_time": S("lastActiveSyncTime"),
156+
"peer_instance": S("peerInstance"),
157+
"state": S("state"),
158+
"state_reasons": S("stateReasons", default=[]),
159+
}
160+
last_active_sync_time: Optional[datetime] = field(default=None)
161+
peer_instance: Optional[str] = field(default=None)
162+
state: Optional[str] = field(default=None)
163+
state_reasons: Optional[List[str]] = field(default=None)
164+
165+
166+
@define(eq=False, slots=False)
167+
class GcpReplication:
168+
kind: ClassVar[str] = "gcp_replication"
169+
mapping: ClassVar[Dict[str, Bender]] = {
170+
"replicas": S("replicas", default=[]) >> ForallBend(GcpReplicaConfig.mapping),
171+
"role": S("role"),
172+
}
173+
replicas: Optional[List[GcpReplicaConfig]] = field(default=None)
174+
role: Optional[str] = field(default=None)
175+
176+
177+
@define(eq=False, slots=False)
178+
class GcpFilestoreInstance(GcpResource, BaseNetworkShare):
179+
kind: ClassVar[str] = "gcp_filestore_instance"
180+
_kind_display: ClassVar[str] = "GCP Filestore Instance"
181+
_kind_description: ClassVar[str] = (
182+
"GCP Filestore Instance is a fully managed file storage service that provides scalable and high-performance"
183+
" file systems for applications running on Google Cloud."
184+
)
185+
_docs_url: ClassVar[str] = "https://cloud.google.com/filestore/docs/instances"
186+
_kind_service: ClassVar[Optional[str]] = "filestore"
187+
_metadata: ClassVar[Dict[str, Any]] = {"icon": "network_share", "group": "storage"}
188+
_reference_kinds: ClassVar[ModelReference] = {
189+
"successors": {
190+
"default": [
191+
"gcp_filestore_instance_snapshot",
192+
],
193+
},
194+
}
195+
api_spec: ClassVar[GcpApiSpec] = GcpApiSpec(
196+
service="file",
197+
version="v1",
198+
accessors=["projects", "locations", "instances"],
199+
action="list",
200+
request_parameter={"parent": "projects/{project}/locations/-"},
201+
request_parameter_in={"project"},
202+
response_path="instances",
203+
response_regional_sub_path=None,
204+
)
205+
mapping: ClassVar[Dict[str, Bender]] = {
206+
"id": S("name").or_else(S("id")).or_else(S("selfLink")),
207+
"name": S("name"),
208+
"ctime": S("creationTimestamp"),
209+
"description": S("description"),
210+
"link": S("selfLink"),
211+
"label_fingerprint": S("labelFingerprint"),
212+
"deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping),
213+
"configurable_performance_enabled": S("configurablePerformanceEnabled"),
214+
"create_time": S("createTime"),
215+
"deletion_protection_enabled": S("deletionProtectionEnabled"),
216+
"deletion_protection_reason": S("deletionProtectionReason"),
217+
"etag": S("etag"),
218+
"file_shares": S("fileShares", default=[]) >> ForallBend(GcpFileShareConfig.mapping),
219+
"kms_key_name": S("kmsKeyName"),
220+
"networks": S("networks", default=[]) >> ForallBend(GcpNetworkConfig.mapping),
221+
"performance_config": S("performanceConfig", default={}) >> Bend(GcpPerformanceConfig.mapping),
222+
"performance_limits": S("performanceLimits", default={}) >> Bend(GcpPerformanceLimits.mapping),
223+
"protocol": S("protocol"),
224+
"replication": S("replication", default={}) >> Bend(GcpReplication.mapping),
225+
"satisfies_pzi": S("satisfiesPzi"),
226+
"satisfies_pzs": S("satisfiesPzs"),
227+
"state": S("state"),
228+
"status_message": S("statusMessage"),
229+
"suspension_reasons": S("suspensionReasons", default=[]),
230+
"tags": S("tags", default={}),
231+
"tier": S("tier"),
232+
}
233+
configurable_performance_enabled: Optional[bool] = field(default=None)
234+
create_time: Optional[datetime] = field(default=None)
235+
deletion_protection_enabled: Optional[bool] = field(default=None)
236+
deletion_protection_reason: Optional[str] = field(default=None)
237+
etag: Optional[str] = field(default=None)
238+
file_shares: Optional[List[GcpFileShareConfig]] = field(default=None)
239+
kms_key_name: Optional[str] = field(default=None)
240+
networks: Optional[List[GcpNetworkConfig]] = field(default=None)
241+
performance_config: Optional[GcpPerformanceConfig] = field(default=None)
242+
performance_limits: Optional[GcpPerformanceLimits] = field(default=None)
243+
protocol: Optional[str] = field(default=None)
244+
replication: Optional[GcpReplication] = field(default=None)
245+
satisfies_pzi: Optional[bool] = field(default=None)
246+
satisfies_pzs: Optional[bool] = field(default=None)
247+
state: Optional[str] = field(default=None)
248+
status_message: Optional[str] = field(default=None)
249+
suspension_reasons: Optional[List[str]] = field(default=None)
250+
tier: Optional[str] = field(default=None)
251+
252+
@classmethod
253+
def called_collect_apis(cls) -> List[GcpApiSpec]:
254+
return [
255+
cls.api_spec,
256+
GcpApiSpec(
257+
service="file",
258+
version="v1",
259+
accessors=["projects", "locations", "instances", "snapshots"],
260+
action="list",
261+
request_parameter={"parent": "projects/{project}/locations/{location}/instances/{instanceId}"},
262+
request_parameter_in={"project", "location", "instanceId"},
263+
response_path="snapshots",
264+
response_regional_sub_path=None,
265+
),
266+
]
267+
268+
def post_process(self, graph_builder: GraphBuilder, source: Json) -> None:
269+
def collect_snapshots() -> None:
270+
spec = GcpApiSpec(
271+
service="file",
272+
version="v1",
273+
accessors=["projects", "locations", "instances", "snapshots"],
274+
action="list",
275+
request_parameter={"parent": f"{self.id}"},
276+
request_parameter_in=set(),
277+
response_path="snapshots",
278+
response_regional_sub_path=None,
279+
)
280+
with GcpErrorHandler(
281+
spec.action,
282+
graph_builder.error_accumulator,
283+
spec.service,
284+
graph_builder.region.safe_name if graph_builder.region else None,
285+
set(),
286+
f" in {graph_builder.project.id} kind {GcpFilestoreInstanceSnapshot.kind}",
287+
):
288+
items = graph_builder.client.list(spec)
289+
snapshots = GcpFilestoreInstanceSnapshot.collect(items, graph_builder)
290+
for snapshot in snapshots:
291+
graph_builder.add_edge(self, node=snapshot)
292+
log.info(f"[GCP:{graph_builder.project.id}] finished collecting: {GcpFilestoreInstanceSnapshot.kind}")
293+
294+
graph_builder.submit_work(collect_snapshots)
295+
296+
297+
@define(eq=False, slots=False)
298+
class GcpFilestoreInstanceSnapshot(GcpResource):
299+
# collected via GcpFilestoreInstance()
300+
kind: ClassVar[str] = "gcp_filestore_instance_snapshot"
301+
_kind_display: ClassVar[str] = "GCP Filestore Snapshot"
302+
_kind_description: ClassVar[str] = (
303+
"GCP Filestore Snapshot is a point-in-time copy of a Filestore instance, allowing you to restore"
304+
" data to a previous state or create new instances from the snapshot."
305+
)
306+
_docs_url: ClassVar[str] = "https://cloud.google.com/filestore/docs/snapshots"
307+
_kind_service: ClassVar[Optional[str]] = "filestore"
308+
_metadata: ClassVar[Dict[str, Any]] = {"icon": "snapshot", "group": "storage"}
309+
mapping: ClassVar[Dict[str, Bender]] = {
310+
"id": S("name").or_else(S("id")).or_else(S("selfLink")),
311+
"name": S("name"),
312+
"ctime": S("creationTimestamp"),
313+
"description": S("description"),
314+
"link": S("selfLink"),
315+
"label_fingerprint": S("labelFingerprint"),
316+
"deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping),
317+
"create_time": S("createTime"),
318+
"filesystem_used_bytes": S("filesystemUsedBytes"),
319+
"state": S("state"),
320+
"tags": S("tags", default={}),
321+
}
322+
create_time: Optional[datetime] = field(default=None)
323+
filesystem_used_bytes: Optional[str] = field(default=None)
324+
state: Optional[str] = field(default=None)
325+
326+
327+
resources: List[Type[GcpResource]] = [GcpFilestoreBackup, GcpFilestoreInstance, GcpFilestoreInstanceSnapshot]

‎plugins/gcp/test/files/filestore_backup.json‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
{
2+
"backups": [
3+
{
4+
"id": "projects/sample-project/locations/us-central1/backups/backup-1",
5+
"name": "backup-1",
6+
"ctime": "2024-11-11T08:30:00Z",
7+
"description": "Daily backup for critical data",
8+
"link": "https://www.googleapis.com/filestore/v1/projects/sample-project/locations/us-central1/backups/backup-1",
9+
"label_fingerprint": "abc123",
10+
"deprecation_status": {
11+
"state": "ACTIVE",
12+
"deleted": null,
13+
"deprecated": null,
14+
"obsolete": null,
15+
"replacement": null
16+
},
17+
"capacity_gb": "1024",
18+
"create_time": "2024-11-11T08:30:00Z",
19+
"download_bytes": "5368709120",
20+
"file_system_protocol": "NFSv3",
21+
"kms_key": "projects/sample-project/locations/us-central1/keyRings/sample-ring/cryptoKeys/sample-key",
22+
"satisfies_pzi": true,
23+
"satisfies_pzs": false,
24+
"source_file_share": "share-1",
25+
"source_instance": "projects/sample-project/locations/us-central1/instances/instance-1",
26+
"source_instance_tier": "STANDARD",
27+
"state": "READY",
28+
"storage_bytes": "5368709120"
29+
}
30+
]
31+
}

0 commit comments

Comments
 (0)