[resotocore][feat] Maintain history only for configuration changes (#…

…1910)

plugins/aws/resoto_plugin_aws/resource/ecr.py

@@ -7,6 +7,7 @@
 
 from resoto_plugin_aws.resource.base import AwsResource, AwsApiSpec, GraphBuilder
 from resoto_plugin_aws.utils import ToDict
+from resotolib.json import sort_json
 from resotolib.json_bender import Bender, S, Bend
 from resotolib.types import Json
 
@@ -61,7 +62,7 @@ def fetch_lifecycle_policy(repository: AwsEcrRepository) -> None:
                     repositoryName=repository.name,
                     expected_errors=["LifecyclePolicyNotFoundException"],
                 ):
-                    repository.lifecycle_policy = json.loads(policy["lifecyclePolicyText"])
+                    repository.lifecycle_policy = sort_json(json.loads(policy["lifecyclePolicyText"]), sort_list=True)
 
         def collect(visibility: str, spec: AwsApiSpec) -> None:
             try:

plugins/aws/resoto_plugin_aws/resource/efs.py

@@ -10,6 +10,7 @@
 from resoto_plugin_aws.utils import ToDict
 from resotolib.baseresources import ModelReference, BaseNetworkShare
 from resotolib.graph import Graph
+from resotolib.json import sort_json
 from resotolib.json_bender import Bender, S, F, Bend
 from resotolib.types import Json
 
@@ -147,7 +148,7 @@ def fetch_file_system_policy(fs: AwsEfsFileSystem) -> None:
                     FileSystemId=fs.id,
                     expected_errors=["PolicyNotFound"],
                 ):
-                    fs.file_system_policy = json.loads(policy["Policy"])
+                    fs.file_system_policy = sort_json(json.loads(policy["Policy"]), sort_list=True)
 
         for js in js_list:
             if instance := cls.from_api(js, builder):

plugins/aws/resoto_plugin_aws/resource/glacier.py

@@ -9,6 +9,7 @@
 from resoto_plugin_aws.resource.sns import AwsSnsTopic
 from resotolib.baseresources import EdgeType, ModelReference
 from resotolib.graph import Graph
+from resotolib.json import sort_json
 from resotolib.json_bender import S, Bend, Bender, ForallBend
 from resotolib.types import Json
 
@@ -258,7 +259,7 @@ def access_policy(vault: AwsGlacierVault) -> None:
                 expected_errors=["ResourceNotFoundException"],
             )
             if response and (policy_string := response.get("Policy")):
-                vault.glacier_access_policy = json.loads(policy_string)
+                vault.glacier_access_policy = sort_json(json.loads(policy_string), sort_list=True)
 
         for vault in source:
             if vault_instance := cls.from_api(vault, builder):

plugins/aws/resoto_plugin_aws/resource/iam.py

@@ -21,7 +21,7 @@
 )
 from resotolib.graph import Graph
 from resotolib.json import value_in_path
-from resotolib.json_bender import Bender, S, Bend, AsDate, Sort, bend, ForallBend, F
+from resotolib.json_bender import Bender, S, Bend, AsDate, Sort, bend, ForallBend, F, Sorted
 from resotolib.types import Json
 from resotolib.utils import parse_utc, utc
 
@@ -62,7 +62,10 @@ class AwsIamPolicyDetail:
         "IAM Policy Detail provides information about the permissions and access"
         " control settings defined in an IAM policy."
     )
-    mapping: ClassVar[Dict[str, Bender]] = {"policy_name": S("PolicyName"), "policy_document": S("PolicyDocument")}
+    mapping: ClassVar[Dict[str, Bender]] = {
+        "policy_name": S("PolicyName"),
+        "policy_document": S("PolicyDocument") >> Sorted(sort_list=True),
+    }
     policy_name: Optional[str] = field(default=None)
     policy_document: Optional[Json] = field(default=None)
 
@@ -126,7 +129,7 @@ class AwsIamRole(AwsResource):
         "atime": (S("RoleLastUsed") >> Sort(S("LastUsedDate") >> AsDate()))[-1]["LastUsedDate"],
         "path": S("Path"),
         "arn": S("Arn"),
-        "role_assume_role_policy_document": S("AssumeRolePolicyDocument"),
+        "role_assume_role_policy_document": S("AssumeRolePolicyDocument") >> Sorted(sort_list=True),
         "description": S("Description"),
         "role_max_session_duration": S("MaxSessionDuration"),
         "role_permissions_boundary": S("PermissionsBoundary") >> Bend(AwsIamAttachedPermissionsBoundary.mapping),
@@ -135,10 +138,10 @@ class AwsIamRole(AwsResource):
     }
     path: Optional[str] = field(default=None)
     description: Optional[str] = field(default=None)
-    role_assume_role_policy_document: Optional[Any] = field(default=None)
+    role_assume_role_policy_document: Optional[Json] = field(default=None)
     role_max_session_duration: Optional[int] = field(default=None)
     role_permissions_boundary: Optional[AwsIamAttachedPermissionsBoundary] = field(default=None)
-    role_last_used: Optional[AwsIamRoleLastUsed] = field(default=None)
+    role_last_used: Optional[AwsIamRoleLastUsed] = field(default=None, metadata=dict(ignore_history=True))
     role_policies: List[AwsIamPolicyDetail] = field(factory=list)
 
     def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None:
@@ -287,7 +290,7 @@ class AwsIamPolicyVersion:
         " access control for AWS resources."
     )
     mapping: ClassVar[Dict[str, Bender]] = {
-        "document": S("Document"),
+        "document": S("Document") >> Sorted(sort_list=True),
         "version_id": S("VersionId"),
         "is_default_version": S("IsDefaultVersion"),
         "create_date": S("CreateDate"),

plugins/aws/resoto_plugin_aws/resource/kms.py

@@ -6,6 +6,7 @@
 from resotolib.baseresources import BaseAccessKey
 from resoto_plugin_aws.utils import ToDict
 from resotolib.graph import Graph
+from resotolib.json import sort_json
 from resotolib.json_bender import Bend, Bender, S, ForallBend, bend
 from resotolib.types import Json
 
@@ -154,7 +155,7 @@ def fetch_key_policy(key: AwsKmsKey) -> None:
                     expected_errors=["NotFoundException"],
                 )
                 if key_policy is not None:
-                    key.kms_key_policy = json.loads(key_policy)
+                    key.kms_key_policy = sort_json(json.loads(key_policy), sort_list=True)
 
         def add_rotation_status(key: AwsKmsKey) -> None:
             with builder.suppress(f"{service_name}.get-key-rotation-status"):

plugins/aws/resoto_plugin_aws/resource/opensearch.py

@@ -9,7 +9,7 @@
 from resoto_plugin_aws.resource.cognito import AwsCognitoUserPool
 from resoto_plugin_aws.resource.ec2 import AwsEc2Subnet, AwsEc2SecurityGroup, AwsEc2Vpc, AwsEc2InstanceType
 from resoto_plugin_aws.utils import ToDict
-from resotolib.json_bender import Bender, S, Bend, ParseJson
+from resotolib.json_bender import Bender, S, Bend, ParseJson, Sorted
 from resotolib.types import Json
 
 log = logging.getLogger("resoto.plugins.aws")
@@ -266,7 +266,7 @@ class AwsOpenSearchDomain(AwsResource):
         "engine_version": S("EngineVersion"),
         "cluster_config": S("ClusterConfig") >> Bend(AwsOpenSearchClusterConfig.mapping),
         "ebs_options": S("EBSOptions") >> Bend(AwsOpenSearchEBSOptions.mapping),
-        "access_policies": S("AccessPolicies") >> ParseJson(),
+        "access_policies": S("AccessPolicies") >> ParseJson() >> Sorted(sort_list=True),
         "ip_address_type": S("IPAddressType"),
         "snapshot_options": S("SnapshotOptions", "AutomatedSnapshotStartHour"),
         "vpc_options": S("VPCOptions") >> Bend(AwsOpenSearchVPCDerivedInfo.mapping),

plugins/aws/resoto_plugin_aws/resource/s3.py

@@ -10,7 +10,7 @@
 from resoto_plugin_aws.utils import tags_as_dict
 from resotolib.baseresources import BaseBucket, PhantomBaseResource, ModelReference
 from resotolib.graph import Graph
-from resotolib.json import is_empty
+from resotolib.json import is_empty, sort_json
 from resotolib.json_bender import Bender, S, bend, Bend, ForallBend
 from resotolib.types import Json
 
@@ -225,7 +225,7 @@ def add_bucket_policy(bck: AwsS3Bucket) -> None:
                     Bucket=bck.name,
                     expected_errors=["NoSuchBucketPolicy", "NoSuchBucket"],
                 ):
-                    bck.bucket_policy = json_loads(raw_policy)  # type: ignore # this is a string
+                    bck.bucket_policy = sort_json(json_loads(raw_policy), sort_list=True)  # type: ignore
 
         def add_bucket_versioning(bck: AwsS3Bucket) -> None:
             with builder.suppress(f"{service_name}.get-bucket-versioning"):

plugins/aws/resoto_plugin_aws/resource/sns.py

@@ -7,7 +7,7 @@
 from resoto_plugin_aws.utils import ToDict
 from resotolib.baseresources import EdgeType, ModelReference
 from resotolib.graph import Graph
-from resotolib.json_bender import F, Bender, S, bend, ParseJson
+from resotolib.json_bender import F, Bender, S, bend, ParseJson, Sorted
 from resotolib.types import Json
 
 service_name = "sns"
@@ -38,7 +38,7 @@ class AwsSnsTopic(AwsResource):
         "topic_subscriptions_confirmed": S("SubscriptionsConfirmed") >> F(lambda x: int(x)),
         "topic_subscriptions_deleted": S("SubscriptionsDeleted") >> F(lambda x: int(x)),
         "topic_subscriptions_pending": S("SubscriptionsPending") >> F(lambda x: int(x)),
-        "topic_policy": S("Policy") >> ParseJson(),
+        "topic_policy": S("Policy") >> ParseJson() >> Sorted(sort_list=True),
         "topic_delivery_policy": S("DeliveryPolicy"),
         "topic_effective_delivery_policy": S("EffectiveDeliveryPolicy"),
         "topic_owner": S("Owner"),

plugins/aws/resoto_plugin_aws/resource/sqs.py

@@ -8,7 +8,7 @@
 from resoto_plugin_aws.resource.kms import AwsKmsKey
 from resotolib.baseresources import ModelReference
 from resotolib.graph import Graph
-from resotolib.json_bender import F, Bender, S, AsInt, AsBool, Bend, ParseJson
+from resotolib.json_bender import F, Bender, S, AsInt, AsBool, Bend, ParseJson, Sorted
 from resotolib.types import Json
 from resotolib.utils import utc_str
 
@@ -57,7 +57,7 @@ class AwsSqsQueue(AwsResource):
         "sqs_approximate_number_of_messages": S("ApproximateNumberOfMessages") >> AsInt(),
         "sqs_approximate_number_of_messages_not_visible": S("ApproximateNumberOfMessagesNotVisible") >> AsInt(),
         "sqs_approximate_number_of_messages_delayed": S("ApproximateNumberOfMessagesDelayed") >> AsInt(),
-        "sqs_policy": S("Policy") >> ParseJson(),
+        "sqs_policy": S("Policy") >> ParseJson() >> Sorted(sort_list=True),
         "sqs_redrive_policy": S("RedrivePolicy") >> ParseJson() >> Bend(AwsSqsRedrivePolicy.mapping),
         "sqs_fifo_queue": S("FifoQueue") >> AsBool(),
         "sqs_content_based_deduplication": S("ContentBasedDeduplication") >> AsBool(),

plugins/aws/resoto_plugin_aws/resource/ssm.py

@@ -68,7 +68,7 @@ class AwsSSMInstance(AwsResource):
     }
     instance_id: Optional[str] = field(default=None, metadata={"description": "The managed node ID."})  # fmt: skip
     ping_status: Optional[str] = field(default=None, metadata={"description": "Connection status of SSM Agent.   The status Inactive has been deprecated and is no longer in use."})  # fmt: skip
-    last_ping: Optional[datetime] = field(default=None, metadata={"description": "The date and time when the agent last pinged the Systems Manager service."})  # fmt: skip
+    last_ping: Optional[datetime] = field(default=None, metadata={"description": "The date and time when the agent last pinged the Systems Manager service.", "ignore_history": True})  # fmt: skip
     agent_version: Optional[str] = field(default=None, metadata={"description": "The version of SSM Agent running on your Linux managed node."})  # fmt: skip
     is_latest_version: Optional[bool] = field(default=None, metadata={"description": "Indicates whether the latest version of SSM Agent is running on your Linux managed node. This field doesn't indicate whether or not the latest version is installed on Windows managed nodes, because some older versions of Windows Server use the EC2Config service to process Systems Manager requests."})  # fmt: skip
     platform_type: Optional[str] = field(default=None, metadata={"description": "The operating system platform type."})  # fmt: skip

plugins/dockerhub/resoto_plugin_dockerhub/resources.py

@@ -1,4 +1,6 @@
 from datetime import datetime
+
+from attr import field
 from attrs import define
 from typing import Optional, ClassVar, List, Dict
 from resotolib.graph import Graph
@@ -42,7 +44,7 @@ class DockerHubRepository(DockerHubResource, BaseResource):
     repository_type: Optional[str] = None
     is_private: Optional[bool] = None
     star_count: Optional[int] = None
-    pull_count: Optional[int] = None
+    pull_count: Optional[int] = field(default=None, metadata=dict(ignore_history=True))
     affiliation: Optional[str] = None
     media_types: Optional[List[str]] = None