Skip to content

Commit ebb67be

Browse files
1101-1aquamatthias
andauthored
[aws][fix] Collect and connect Inspector resources properly (#2253)
Co-authored-by: Matthias Veit <matthias_veit@yahoo.de>
1 parent 2022f48 commit ebb67be

File tree

14 files changed

+483
-14
lines changed

14 files changed

+483
-14
lines changed

fixlib/fixlib/baseresources.py

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,7 @@ class Finding:
259259
severity: Severity = Severity.medium
260260
description: Optional[str] = None
261261
remediation: Optional[str] = None
262-
created_at: Optional[datetime] = None
262+
updated_at: Optional[datetime] = None
263263
details: Optional[Json] = None
264264

265265

@@ -409,6 +409,13 @@ def log(self, msg: str, data: Optional[Any] = None, exception: Optional[Exceptio
409409
self.__log.append(log_entry)
410410
self._changes.add("log")
411411

412+
def add_finding(self, provider: str, finding: Finding) -> None:
413+
for assessment in self._assessments:
414+
if assessment.provider == provider:
415+
assessment.findings.append(finding)
416+
return
417+
self._assessments.append(Assessment(provider=provider, findings=[finding]))
418+
412419
def add_change(self, change: str) -> None:
413420
self._changes.add(change)
414421

plugins/aws/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ clean-test: ## remove test and coverage artifacts
2929

3030
lint: ## static code analysis
3131
black --line-length 120 --check fix_plugin_aws test
32-
flake8 fix_plugin_aws
32+
flake8 fix_plugin_aws test
3333
mypy --python-version 3.12 --strict --install-types fix_plugin_aws test
3434

3535
test: ## run tests quickly with the default Python

plugins/aws/fix_plugin_aws/collector.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
backup,
5151
bedrock,
5252
scp,
53+
inspector,
5354
)
5455
from fix_plugin_aws.resource.base import (
5556
AwsAccount,
@@ -118,6 +119,7 @@
118119
+ backup.resources
119120
+ amazonq.resources
120121
+ bedrock.resources
122+
+ inspector.resources
121123
)
122124
all_resources: List[Type[AwsResource]] = global_resources + regional_resources
123125

@@ -244,6 +246,10 @@ def get_last_run() -> Optional[datetime]:
244246
)
245247
shared_queue.wait_for_submitted_work()
246248

249+
# call all registered after collect hooks
250+
for after_collect in global_builder.after_collect_actions:
251+
after_collect()
252+
247253
# connect nodes
248254
log.info(f"[Aws:{self.account.id}] Connect resources and create edges.")
249255
for node, data in list(self.graph.nodes(data=True)):

plugins/aws/fix_plugin_aws/resource/backup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ class AwsBackupProtectedResource(AwsResource):
175175
}
176176
api_spec: ClassVar[AwsApiSpec] = AwsApiSpec("backup", "list-protected-resources", "Results")
177177
mapping: ClassVar[Dict[str, Bender]] = {
178-
"id": S("ResourceArn") >> F(lambda arn: arn.rsplit("/")[1]),
178+
"id": S("ResourceArn") >> F(AwsResource.id_from_arn),
179179
"name": S("ResourceName"),
180180
"resource_arn": S("ResourceArn"),
181181
"resource_type": S("ResourceType"),

plugins/aws/fix_plugin_aws/resource/base.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -453,6 +453,7 @@ def __init__(
453453
graph_nodes_access: Optional[RWLock] = None,
454454
graph_edges_access: Optional[RWLock] = None,
455455
last_run_started_at: Optional[datetime] = None,
456+
after_collect_actions: Optional[List[Callable[[], Any]]] = None,
456457
) -> None:
457458
self.graph = graph
458459
self.cloud = cloud
@@ -469,6 +470,7 @@ def __init__(
469470
self.last_run_started_at = last_run_started_at
470471
self.created_at = utc()
471472
self.__builder_cache = {region.safe_name: self}
473+
self.after_collect_actions = after_collect_actions if after_collect_actions is not None else []
472474

473475
if last_run_started_at:
474476
now = utc()
@@ -705,6 +707,7 @@ def for_region(self, region: AwsRegion) -> GraphBuilder:
705707
self.graph_nodes_access,
706708
self.graph_edges_access,
707709
self.last_run_started_at,
710+
self.after_collect_actions,
708711
)
709712
self.__builder_cache[region.safe_name] = builder
710713
return builder

plugins/aws/fix_plugin_aws/resource/ec2.py

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
import base64
2-
from functools import partial
2+
import copy
33
import logging
44
from contextlib import suppress
55
from datetime import datetime, timedelta
6+
from functools import partial
67
from typing import ClassVar, Dict, Optional, List, Type, Any
7-
import copy
88

99
from attrs import define, field
10-
from fix_plugin_aws.aws_client import AwsClient
1110

11+
from fix_plugin_aws.aws_client import AwsClient
1212
from fix_plugin_aws.resource.base import AwsResource, GraphBuilder, AwsApiSpec, get_client
1313
from fix_plugin_aws.resource.cloudwatch import (
1414
AwsCloudwatchQuery,
@@ -18,9 +18,9 @@
1818
operations_to_iops,
1919
normalizer_factory,
2020
)
21+
from fix_plugin_aws.resource.iam import AwsIamInstanceProfile
2122
from fix_plugin_aws.resource.kms import AwsKmsKey
2223
from fix_plugin_aws.resource.s3 import AwsS3Bucket
23-
from fix_plugin_aws.resource.iam import AwsIamInstanceProfile
2424
from fix_plugin_aws.utils import ToDict, TagsValue
2525
from fixlib.baseresources import (
2626
BaseInstance,
@@ -49,7 +49,6 @@
4949
from fixlib.json_bender import Bender, S, Bend, ForallBend, bend, MapEnum, F, K, StripNones
5050
from fixlib.types import Json
5151

52-
5352
# region InstanceType
5453
from fixlib.utils import utc
5554

plugins/aws/fix_plugin_aws/resource/ecr.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
import json
22
import logging
3-
from typing import ClassVar, Dict, Optional, List, Tuple, Type, Any
43
from json import loads as json_loads
4+
from typing import ClassVar, Dict, Optional, List, Tuple, Type, Any
55

66
from attrs import define, field
77
from boto3.exceptions import Boto3Error
88

99
from fix_plugin_aws.resource.base import AwsResource, AwsApiSpec, GraphBuilder
1010
from fix_plugin_aws.utils import ToDict
11-
from fixlib.baseresources import HasResourcePolicy, PolicySource, PolicySourceKind
11+
from fixlib.baseresources import HasResourcePolicy, ModelReference, PolicySource, PolicySourceKind
1212
from fixlib.json import sort_json
1313
from fixlib.json_bender import Bender, S, Bend
1414
from fixlib.types import Json
@@ -34,6 +34,7 @@ class AwsEcrRepository(AwsResource, HasResourcePolicy):
3434
_kind_service: ClassVar[Optional[str]] = service_name
3535
_metadata: ClassVar[Dict[str, Any]] = {"icon": "repository", "group": "compute"}
3636
_aws_metadata: ClassVar[Dict[str, Any]] = {"provider_link_tpl": "https://{region_id}.console.aws.amazon.com/ecr/repositories/{name}?region={region}", "arn_tpl": "arn:{partition}:ecr:{region}:{account}:repository/{name}"} # fmt: skip
37+
_reference_kinds: ClassVar[ModelReference] = {}
3738
api_spec: ClassVar[AwsApiSpec] = AwsApiSpec("ecr", "describe-repositories", "repositories")
3839
public_spec: ClassVar[AwsApiSpec] = AwsApiSpec("ecr-public", "describe-repositories", "repositories")
3940
mapping: ClassVar[Dict[str, Bender]] = {
Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
import logging
2+
from datetime import datetime
3+
from functools import partial
4+
from typing import ClassVar, Dict, Optional, List, Tuple, Type, Any
5+
6+
from attrs import define, field
7+
from boto3.exceptions import Boto3Error
8+
9+
from fix_plugin_aws.resource.base import AwsResource, AwsApiSpec, GraphBuilder
10+
from fix_plugin_aws.resource.ec2 import AwsEc2Instance
11+
from fix_plugin_aws.resource.ecr import AwsEcrRepository
12+
from fix_plugin_aws.resource.lambda_ import AwsLambdaFunction
13+
from fixlib.baseresources import PhantomBaseResource, Severity, Finding
14+
from fixlib.json_bender import Bender, S, ForallBend, Bend, F
15+
from fixlib.types import Json
16+
17+
log = logging.getLogger("fix.plugins.aws")
18+
service_name = "inspector2"
19+
20+
amazon_inspector = "amazon_inspector"
21+
22+
23+
@define(eq=False, slots=False)
24+
class AwsInspectorRecommendation:
25+
kind: ClassVar[str] = "aws_inspector_recommendation"
26+
mapping: ClassVar[Dict[str, Bender]] = {"url": S("Url"), "text": S("text")}
27+
url: Optional[str] = field(default=None, metadata={"description": "The URL address to the CVE remediation recommendations."}) # fmt: skip
28+
text: Optional[str] = field(default=None, metadata={"description": "The recommended course of action to remediate the finding."}) # fmt: skip
29+
30+
31+
@define(eq=False, slots=False)
32+
class AwsInspectorRemediation:
33+
kind: ClassVar[str] = "aws_inspector_remediation"
34+
mapping: ClassVar[Dict[str, Bender]] = {
35+
"recommendation": S("recommendation") >> Bend(AwsInspectorRecommendation.mapping)
36+
}
37+
recommendation: Optional[AwsInspectorRecommendation] = field(default=None, metadata={"description": "An object that contains information about the recommended course of action to remediate the finding."}) # fmt: skip
38+
39+
40+
@define(eq=False, slots=False)
41+
class AwsInspectorResource:
42+
kind: ClassVar[str] = "aws_inspector_resource"
43+
mapping: ClassVar[Dict[str, Bender]] = {
44+
# "details": S("details") # not used
45+
"id": S("id"),
46+
"partition": S("partition"),
47+
"region": S("region"),
48+
"type": S("type"),
49+
}
50+
id: Optional[str] = field(default=None, metadata={"description": "The ID of the resource."}) # fmt: skip
51+
partition: Optional[str] = field(default=None, metadata={"description": "The partition of the resource."}) # fmt: skip
52+
region: Optional[str] = field(default=None, metadata={"description": "The Amazon Web Services Region the impacted resource is located in."}) # fmt: skip
53+
type: Optional[str] = field(default=None, metadata={"description": "The type of resource."}) # fmt: skip
54+
55+
56+
@define(eq=False, slots=False)
57+
class AwsInspectorFinding(AwsResource, PhantomBaseResource):
58+
kind: ClassVar[str] = "aws_inspector_finding"
59+
api_spec: ClassVar[AwsApiSpec] = AwsApiSpec(service_name, "list-findings")
60+
_model_export: ClassVar[bool] = False # do not export this class, since there will be no instances of it
61+
mapping: ClassVar[Dict[str, Bender]] = {
62+
"id": S("findingArn") >> F(AwsResource.id_from_arn),
63+
"name": S("title"),
64+
"mtime": S("updatedAt"),
65+
"arn": S("findingArn"),
66+
"aws_account_id": S("awsAccountId"),
67+
"description": S("description"),
68+
"epss": S("epss", "score"),
69+
"exploit_available": S("exploitAvailable"),
70+
"exploitability_details": S("exploitabilityDetails", "lastKnownExploitAt"),
71+
"finding_arn": S("findingArn"),
72+
"first_observed_at": S("firstObservedAt"),
73+
"fix_available": S("fixAvailable"),
74+
"inspector_score": S("inspectorScore"),
75+
"last_observed_at": S("lastObservedAt"),
76+
"remediation": S("remediation") >> Bend(AwsInspectorRemediation.mapping),
77+
"finding_resources": S("resources", default=[]) >> ForallBend(AwsInspectorResource.mapping),
78+
"finding_severity": S("severity"),
79+
"status": S("status"),
80+
"title": S("title"),
81+
"type": S("type"),
82+
"updated_at": S("updatedAt"),
83+
# available but not used properties:
84+
# "inspector_score_details": S("inspectorScoreDetails")
85+
# "code_vulnerability_details": S("codeVulnerabilityDetails")
86+
# "network_reachability_details": S("networkReachabilityDetails")
87+
# "package_vulnerability_details": S("packageVulnerabilityDetails")
88+
}
89+
aws_account_id: Optional[str] = field(default=None, metadata={"description": "The Amazon Web Services account ID associated with the finding."}) # fmt: skip
90+
description: Optional[str] = field(default=None, metadata={"description": "The description of the finding."}) # fmt: skip
91+
epss: Optional[float] = field(default=None, metadata={"description": "The finding's EPSS score."}) # fmt: skip
92+
exploit_available: Optional[str] = field(default=None, metadata={"description": "If a finding discovered in your environment has an exploit available."}) # fmt: skip
93+
exploitability_details: Optional[datetime] = field(default=None, metadata={"description": "The details of an exploit available for a finding discovered in your environment."}) # fmt: skip
94+
finding_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Number (ARN) of the finding."}) # fmt: skip
95+
first_observed_at: Optional[datetime] = field(default=None, metadata={"description": "The date and time that the finding was first observed."}) # fmt: skip
96+
fix_available: Optional[str] = field(default=None, metadata={"description": "Details on whether a fix is available through a version update. This value can be YES, NO, or PARTIAL. A PARTIAL fix means that some, but not all, of the packages identified in the finding have fixes available through updated versions."}) # fmt: skip
97+
inspector_score: Optional[float] = field(default=None, metadata={"description": "The Amazon Inspector score given to the finding."}) # fmt: skip
98+
last_observed_at: Optional[datetime] = field(default=None, metadata={"description": "The date and time the finding was last observed. This timestamp for this field remains unchanged until a finding is updated."}) # fmt: skip
99+
remediation: Optional[AwsInspectorRemediation] = field(default=None, metadata={"description": "An object that contains the details about how to remediate a finding."}) # fmt: skip
100+
finding_resources: Optional[List[AwsInspectorResource]] = field(factory=list, metadata={"description": "Contains information on the resources involved in a finding. The resource value determines the valid values for type in your request. For more information, see Finding types in the Amazon Inspector user guide."}) # fmt: skip
101+
finding_severity: Optional[str] = field(default=None, metadata={"description": "The severity of the finding. UNTRIAGED applies to PACKAGE_VULNERABILITY type findings that the vendor has not assigned a severity yet. For more information, see Severity levels for findings in the Amazon Inspector user guide."}) # fmt: skip
102+
status: Optional[str] = field(default=None, metadata={"description": "The status of the finding."}) # fmt: skip
103+
title: Optional[str] = field(default=None, metadata={"description": "The title of the finding."}) # fmt: skip
104+
type: Optional[str] = field(default=None, metadata={"description": "The type of the finding. The type value determines the valid values for resource in your request. For more information, see Finding types in the Amazon Inspector user guide."}) # fmt: skip
105+
updated_at: Optional[datetime] = field(default=None, metadata={"description": "The date and time the finding was last updated at."}) # fmt: skip
106+
107+
def parse_finding(self, source: Json) -> Finding:
108+
severity_mapping = {
109+
"INFORMATIONAL": Severity.info,
110+
"LOW": Severity.low,
111+
"MEDIUM": Severity.medium,
112+
"HIGH": Severity.high,
113+
"CRITICAL": Severity.critical,
114+
}
115+
finding_title = self.safe_name
116+
if not self.finding_severity:
117+
finding_severity = Severity.medium
118+
else:
119+
finding_severity = severity_mapping.get(self.finding_severity, Severity.medium)
120+
description = self.description
121+
remediation = ""
122+
if self.remediation and self.remediation.recommendation:
123+
remediation = self.remediation.recommendation.text or ""
124+
updated_at = self.updated_at
125+
details = source.get("packageVulnerabilityDetails", {}) | source.get("codeVulnerabilityDetails", {})
126+
return Finding(finding_title, finding_severity, description, remediation, updated_at, details)
127+
128+
@classmethod
129+
def collect_resources(cls, builder: GraphBuilder) -> None:
130+
def check_type_and_adjust_id(
131+
class_type: Optional[str], resource_id: Optional[str]
132+
) -> Tuple[Optional[Type[Any]], Optional[Dict[str, Any]]]:
133+
if not resource_id or not class_type:
134+
return None, None
135+
match class_type:
136+
case "AWS_LAMBDA_FUNCTION":
137+
# remove lambda's version from arn
138+
lambda_arn = resource_id.rsplit(":", 1)[0]
139+
return AwsLambdaFunction, {"arn": lambda_arn}
140+
case "AWS_EC2_INSTANCE":
141+
return AwsEc2Instance, {"id": resource_id}
142+
case "AWS_ECR_REPOSITORY":
143+
return AwsEcrRepository, {"id": resource_id, "_region": builder.region}
144+
case _:
145+
return None, None
146+
147+
def add_finding(
148+
provider: str, finding: Finding, clazz: Optional[Type[AwsResource]] = None, **node: Any
149+
) -> None:
150+
if resource := builder.node(clazz=clazz, **node):
151+
resource.add_finding(provider, finding)
152+
153+
# Default behavior: in case the class has an ApiSpec, call the api and call collect.
154+
log.debug(f"Collecting {cls.__name__} in region {builder.region.name}")
155+
try:
156+
for item in builder.client.list(
157+
aws_service=service_name,
158+
action="list-findings",
159+
result_name="findings",
160+
expected_errors=["AccessDeniedException"],
161+
filterCriteria={"awsAccountId": [{"comparison": "EQUALS", "value": f"{builder.account.id}"}]},
162+
):
163+
if finding := AwsInspectorFinding.from_api(item, builder):
164+
for fr in finding.finding_resources or []:
165+
clazz, res_filter = check_type_and_adjust_id(fr.type, fr.id)
166+
if clazz and res_filter:
167+
# append the finding when all resources have been collected
168+
builder.after_collect_actions.append(
169+
partial(
170+
add_finding,
171+
amazon_inspector,
172+
finding.parse_finding(item),
173+
clazz,
174+
**res_filter,
175+
)
176+
)
177+
except Boto3Error as e:
178+
msg = f"Error while collecting {cls.__name__} in region {builder.region.name}: {e}"
179+
builder.core_feedback.error(msg, log)
180+
raise
181+
except Exception as e:
182+
msg = f"Error while collecting {cls.__name__} in region {builder.region.name}: {e}"
183+
builder.core_feedback.info(msg, log)
184+
raise
185+
186+
187+
resources: List[Type[AwsResource]] = [AwsInspectorFinding]

plugins/aws/fix_plugin_aws/resource/lambda_.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
1-
from datetime import timedelta
21
import json as json_p
32
import logging
43
import re
4+
from datetime import timedelta
55
from typing import ClassVar, Dict, Optional, List, Tuple, Type, Any
66

77
from attrs import define, field
8+
89
from fix_plugin_aws.aws_client import AwsClient
910
from fix_plugin_aws.resource.base import AwsResource, GraphBuilder, AwsApiSpec, parse_json
1011
from fix_plugin_aws.resource.cloudwatch import AwsCloudwatchQuery, normalizer_factory
@@ -19,9 +20,9 @@
1920
PolicySourceKind,
2021
)
2122
from fixlib.graph import Graph
23+
from fixlib.json import sort_json
2224
from fixlib.json_bender import Bender, S, Bend, ForallBend, F, bend
2325
from fixlib.types import Json
24-
from fixlib.json import sort_json
2526

2627
log = logging.getLogger("fix.plugins.aws")
2728

plugins/aws/test/collector_test.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@ def count_kind(clazz: Type[AwsResource]) -> int:
2929
return count
3030

3131
for resource in all_resources:
32+
# there will be no instances of resources that are not exported
33+
if not resource._model_export:
34+
continue
3235
assert count_kind(resource) > 0, f"No instances of {resource.__name__} found"
3336

3437
# make sure all threads have been joined
@@ -106,6 +109,8 @@ def all_base_classes(cls: Type[Any]) -> Set[Type[Any]]:
106109
expected_declared_properties = ["kind", "_kind_display"]
107110
expected_props_in_hierarchy = ["_kind_service", "_metadata"]
108111
for rc in all_resources:
112+
if not rc._model_export:
113+
continue
109114
for prop in expected_declared_properties:
110115
assert prop in rc.__dict__, f"{rc.__name__} missing {prop}"
111116
with_bases = (all_base_classes(rc) | {rc}) - {AwsResource, BaseResource}

0 commit comments

Comments
 (0)