[aws][feat] Ignore datetime for history by default (#2256)

Author: aquamatthias

fixcore/fixcore/model/graph_access.py

@@ -334,19 +334,22 @@ def content_hash(
     def history_hash(js: Json, kind: Kind) -> str:
         sha256 = hashlib.sha256()
 
-        def walk_element(el: JsonElement, el_kind: Kind) -> None:
+        def walk_element(el: JsonElement, el_kind: Kind, maybe_prop: Optional[Property]) -> None:
             if el is None:
                 pass
             elif isinstance(el_kind, ComplexKind):
                 walk_complex(el, el_kind)
             elif isinstance(el_kind, ArrayKind):
                 if isinstance(el, list):
                     for elem in el:
-                        walk_element(elem, el_kind.inner)
+                        walk_element(elem, el_kind.inner, maybe_prop)
             elif isinstance(el_kind, DictionaryKind):
                 if isinstance(el, dict):
                     for _, v in sorted(el.items()):
-                        walk_element(v, el_kind.value_kind)
+                        walk_element(v, el_kind.value_kind, maybe_prop)
+            elif isinstance(el_kind, (DateKind, DateTimeKind)):  # default: ignore, opt-in to keep
+                if maybe_prop and maybe_prop.meta_get("keep_history", bool, False):
+                    sha256.update(str(el).encode("utf-8"))
             elif isinstance(el_kind, SimpleKind):
                 sha256.update(str(el).encode("utf-8"))
 
@@ -358,12 +361,12 @@ def walk_complex(el: JsonElement, el_kind: ComplexKind) -> None:
                         and (prop.name not in PropsToIgnoreForHistory)
                         and (prop_val := el.get(prop.name))
                     ):
-                        walk_element(prop_val, prop_kind)
+                        walk_element(prop_val, prop_kind, prop)
                 if not el_kind.metadata.get("ignore_history"):  # if defined on type, do not walk the hierarchy
                     for base in el_kind.resolved_bases().values():
                         walk_complex(el, base)
 
-        walk_element(js, kind)
+        walk_element(js, kind, None)
         return sha256.hexdigest()[0:8]
 
     @staticmethod

fixcore/tests/fixcore/model/graph_access_test.py

@@ -90,6 +90,15 @@ def test_content_hash() -> None:
     assert sha1 == sha2
 
 
+def test_history_hash(person_model: Model) -> None:
+    address = person_model["Address"]
+    a = {"id": "a1", "zip": "s1", "city": "c1", "mtime": "2021-06-18T10:31:34Z"}
+    b = {"id": "a1", "zip": "s1", "city": "c1", "mtime": "2022-06-18T10:31:34Z"}
+    c = {"id": "a1", "zip": "s2", "city": "c2", "mtime": "2022-06-18T10:31:34Z"}
+    assert GraphBuilder.history_hash(a, address) == GraphBuilder.history_hash(b, address)
+    assert GraphBuilder.history_hash(a, address) != GraphBuilder.history_hash(c, address)
+
+
 def test_root(graph_access: GraphAccess) -> None:
     assert graph_access.root() == "1"
 

plugins/aws/fix_plugin_aws/resource/dynamodb.py

@@ -340,8 +340,8 @@ class AwsDynamoDbPointInTimeRecovery:
         "latest_restorable_date_time": S("LatestRestorableDateTime"),
     }
     status: Optional[str] = field(default=None, metadata={"description": "The current state of point in time recovery:    ENABLED - Point in time recovery is enabled.    DISABLED - Point in time recovery is disabled."})  # fmt: skip
-    earliest_restorable_date_time: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Specifies the earliest point in time you can restore your table to. You can restore your table to any point in time during the last 35 days."})  # fmt: skip
-    latest_restorable_date_time: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "LatestRestorableDateTime is typically 5 minutes before the current time."})  # fmt: skip
+    earliest_restorable_date_time: Optional[datetime] = field(default=None, metadata={"description": "Specifies the earliest point in time you can restore your table to. You can restore your table to any point in time during the last 35 days."})  # fmt: skip
+    latest_restorable_date_time: Optional[datetime] = field(default=None, metadata={"description": "LatestRestorableDateTime is typically 5 minutes before the current time."})  # fmt: skip
 
 
 @define(eq=False, slots=False)

plugins/aws/fix_plugin_aws/resource/ecs.py

@@ -488,7 +488,7 @@ class AwsEcsTask(EcsTaggable, AwsResource):
     task_capacity_provider_name: Optional[str] = field(default=None)
     task_cluster_arn: Optional[str] = field(default=None)
     task_connectivity: Optional[str] = field(default=None, metadata=dict(ignore_history=True))
-    task_connectivity_at: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True))
+    task_connectivity_at: Optional[datetime] = field(default=None)
     task_container_instance_arn: Optional[str] = field(default=None)
     task_containers: List[AwsEcsContainer] = field(factory=list)
     task_cpu: Optional[str] = field(default=None)
@@ -504,14 +504,14 @@ class AwsEcsTask(EcsTaggable, AwsResource):
     task_overrides: Optional[AwsEcsTaskOverride] = field(default=None)
     task_platform_version: Optional[str] = field(default=None)
     task_platform_family: Optional[str] = field(default=None)
-    task_pull_started_at: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True))
-    task_pull_stopped_at: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True))
-    task_started_at: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True))
-    task_started_by: Optional[str] = field(default=None, metadata=dict(ignore_history=True))
-    task_stop_code: Optional[str] = field(default=None, metadata=dict(ignore_history=True))
-    task_stopped_at: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True))
-    task_stopped_reason: Optional[str] = field(default=None, metadata=dict(ignore_history=True))
-    task_stopping_at: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True))
+    task_pull_started_at: Optional[datetime] = field(default=None)
+    task_pull_stopped_at: Optional[datetime] = field(default=None)
+    task_started_at: Optional[datetime] = field(default=None)
+    task_started_by: Optional[str] = field(default=None)
+    task_stop_code: Optional[str] = field(default=None)
+    task_stopped_at: Optional[datetime] = field(default=None)
+    task_stopped_reason: Optional[str] = field(default=None)
+    task_stopping_at: Optional[datetime] = field(default=None)
     task_definition_arn: Optional[str] = field(default=None)
     task_version: Optional[int] = field(default=None)
     task_ephemeral_storage: Optional[int] = field(default=None)

plugins/aws/fix_plugin_aws/resource/iam.py

@@ -659,7 +659,7 @@ class AwsIamUser(AwsResource, BaseUser, BaseIamPrincipal):
     user_policies: List[AwsIamPolicyDetail] = field(factory=list)
     user_permissions_boundary: Optional[AwsIamAttachedPermissionsBoundary] = field(default=None)
     password_enabled: Optional[bool] = field(default=None)
-    password_last_used: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True))
+    password_last_used: Optional[datetime] = field(default=None)
     password_last_changed: Optional[datetime] = field(default=None)
     password_next_rotation: Optional[datetime] = field(default=None)
     mfa_active: Optional[bool] = field(default=None)

plugins/aws/fix_plugin_aws/resource/rds.py

@@ -464,7 +464,7 @@ class AwsRdsInstance(RdsTaggable, AwsResource, BaseDatabase):
     rds_db_subnet_group: Optional[AwsRdsDBSubnetGroup] = field(default=None)
     rds_preferred_maintenance_window: Optional[str] = field(default=None)
     rds_pending_modified_values: Optional[AwsRdsPendingModifiedValues] = field(default=None)
-    rds_latest_restorable_time: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True))
+    rds_latest_restorable_time: Optional[datetime] = field(default=None)
     rds_multi_az: Optional[bool] = field(default=None)
     rds_auto_minor_version_upgrade: Optional[bool] = field(default=None)
     rds_read_replica_source_db_instance_identifier: Optional[str] = field(default=None)
@@ -973,12 +973,12 @@ class AwsRdsCluster(RdsTaggable, AwsResource, BaseDatabase):
     rds_db_cluster_parameter_group: Optional[str] = field(default=None)
     rds_db_subnet_group_name: Optional[str] = field(default=None)
     rds_automatic_restart_time: Optional[datetime] = field(default=None)
-    rds_earliest_restorable_time: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True))
+    rds_earliest_restorable_time: Optional[datetime] = field(default=None)
     rds_endpoint: Optional[str] = field(default=None)
     rds_reader_endpoint: Optional[str] = field(default=None)
     rds_custom_endpoints: List[str] = field(factory=list)
     rds_multi_az: Optional[bool] = field(default=None)
-    rds_latest_restorable_time: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True))
+    rds_latest_restorable_time: Optional[datetime] = field(default=None)
     rds_port: Optional[int] = field(default=None)
     rds_master_username: Optional[str] = field(default=None)
     rds_db_cluster_option_group_memberships: List[AwsRdsDBClusterOptionGroupStatus] = field(factory=list)

plugins/aws/fix_plugin_aws/resource/secretsmanager.py

@@ -67,7 +67,7 @@ class AwsSecretsManagerSecret(HasResourcePolicy, AwsResource):
     rotation_rules: Optional[AwsSecretsManagerRotationRulesType] = field(default=None, metadata={"description": "A structure that defines the rotation configuration for the secret."})  # fmt: skip
     last_rotated_date: Optional[datetime] = field(default=None, metadata={"description": "The most recent date and time that the Secrets Manager rotation process was successfully completed. This value is null if the secret hasn't ever rotated."})  # fmt: skip
     last_changed_date: Optional[datetime] = field(default=None, metadata={"description": "The last date and time that this secret was modified in any way."})  # fmt: skip
-    last_accessed_date: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "The date that the secret was last accessed in the Region. This field is omitted if the secret has never been retrieved in the Region."})  # fmt: skip
+    last_accessed_date: Optional[datetime] = field(default=None, metadata={"description": "The date that the secret was last accessed in the Region. This field is omitted if the secret has never been retrieved in the Region."})  # fmt: skip
     deleted_date: Optional[datetime] = field(default=None, metadata={"description": "The date and time the deletion of the secret occurred. Not present on active secrets. The secret can be recovered until the number of days in the recovery window has passed, as specified in the RecoveryWindowInDays parameter of the  DeleteSecret  operation."})  # fmt: skip
     next_rotation_date: Optional[datetime] = field(default=None, metadata={"description": "The next rotation is scheduled to occur on or before this date. If the secret isn't configured for rotation, Secrets Manager returns null."})  # fmt: skip
     secret_versions_to_stages: Optional[Dict[str, List[str]]] = field(default=None, metadata={"description": "A list of all of the currently assigned SecretVersionStage staging labels and the SecretVersionId attached to each one. Staging labels are used to keep track of the different versions during the rotation process.  A version that does not have any SecretVersionStage is considered deprecated and subject to deletion. Such versions are not included in this list."})  # fmt: skip

plugins/aws/fix_plugin_aws/resource/ssm.py

@@ -71,7 +71,7 @@ class AwsSSMInstance(AwsResource):
     }
     instance_id: Optional[str] = field(default=None, metadata={"description": "The managed node ID."})  # fmt: skip
     ping_status: Optional[str] = field(default=None, metadata={"description": "Connection status of SSM Agent.   The status Inactive has been deprecated and is no longer in use."})  # fmt: skip
-    last_ping: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "The date and time when the agent last pinged the Systems Manager service."})  # fmt: skip
+    last_ping: Optional[datetime] = field(default=None, metadata={"description": "The date and time when the agent last pinged the Systems Manager service."})  # fmt: skip
     agent_version: Optional[str] = field(default=None, metadata={"description": "The version of SSM Agent running on your Linux managed node."})  # fmt: skip
     is_latest_version: Optional[bool] = field(default=None, metadata={"description": "Indicates whether the latest version of SSM Agent is running on your Linux managed node. This field doesn't indicate whether or not the latest version is installed on Windows managed nodes, because some older versions of Windows Server use the EC2Config service to process Systems Manager requests."})  # fmt: skip
     platform_type: Optional[str] = field(default=None, metadata={"description": "The operating system platform type."})  # fmt: skip
@@ -84,8 +84,8 @@ class AwsSSMInstance(AwsResource):
     ip_address: Optional[str] = field(default=None, metadata={"description": "The IP address of the managed node."})  # fmt: skip
     computer_name: Optional[str] = field(default=None, metadata={"description": "The fully qualified host name of the managed node."})  # fmt: skip
     association_status: Optional[str] = field(default=None, metadata={"description": "The status of the association."})  # fmt: skip
-    last_association_execution_date: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "The date the association was last run."})  # fmt: skip
-    last_successful_association_execution_date: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "The last date the association was successfully run."})  # fmt: skip
+    last_association_execution_date: Optional[datetime] = field(default=None, metadata={"description": "The date the association was last run."})  # fmt: skip
+    last_successful_association_execution_date: Optional[datetime] = field(default=None, metadata={"description": "The last date the association was successfully run."})  # fmt: skip
     association_overview: Optional[AwsSSMInstanceAggregatedAssociationOverview] = field(default=None, metadata={"description": "Information about the association."})  # fmt: skip
     source_id: Optional[str] = field(default=None, metadata={"description": "The ID of the source resource. For IoT Greengrass devices, SourceId is the Thing name."})  # fmt: skip
     source_type: Optional[str] = field(default=None, metadata={"description": "The type of the source resource. For IoT Greengrass devices, SourceType is AWS::IoT::Thing."})  # fmt: skip

plugins/azure/fix_plugin_azure/resource/authorization.py

@@ -102,7 +102,7 @@ class AzureAuthorizationDenyAssignment(MicrosoftResource):
     condition: Optional[str] = field(default=None, metadata={'description': 'The conditions on the deny assignment. This limits the resources it can be assigned to. e.g.: @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:ContainerName] StringEqualsIgnoreCase foo_storage_container '})  # fmt: skip
     condition_version: Optional[str] = field(default=None, metadata={"description": "Version of the condition."})
     created_by: Optional[str] = field(default=None, metadata={'description': 'Id of the user who created the assignment'})  # fmt: skip
-    created_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was created"})  # fmt: skip
+    created_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was created"})  # fmt: skip
     deny_assignment_name: Optional[str] = field(default=None, metadata={'description': 'The display name of the deny assignment.'})  # fmt: skip
     description: Optional[str] = field(default=None, metadata={'description': 'The description of the deny assignment.'})  # fmt: skip
     do_not_apply_to_child_scopes: Optional[bool] = field(default=None, metadata={'description': 'Determines if the deny assignment applies to child scopes. Default value is false.'})  # fmt: skip
@@ -112,7 +112,7 @@ class AzureAuthorizationDenyAssignment(MicrosoftResource):
     principals: Optional[List[AzurePrincipal]] = field(default=None, metadata={'description': 'Array of principals to which the deny assignment applies.'})  # fmt: skip
     scope: Optional[str] = field(default=None, metadata={"description": "The deny assignment scope."})
     updated_by: Optional[str] = field(default=None, metadata={"ignore_history": True, 'description': 'Id of the user who updated the assignment'})  # fmt: skip
-    updated_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was updated"})  # fmt: skip
+    updated_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was updated"})  # fmt: skip
 
 
 @define(eq=False, slots=False)
@@ -179,15 +179,15 @@ class AzureAuthorizationRoleAssignment(MicrosoftResource):
     condition: Optional[str] = field(default=None, metadata={'description': 'The conditions on the role assignment. This limits the resources it can be assigned to. e.g.: @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:ContainerName] StringEqualsIgnoreCase foo_storage_container '})  # fmt: skip
     condition_version: Optional[str] = field(default=None, metadata={'description': 'Version of the condition. Currently the only accepted value is 2.0 '})  # fmt: skip
     created_by: Optional[str] = field(default=None, metadata={'description': 'Id of the user who created the assignment'})  # fmt: skip
-    created_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was created"})  # fmt: skip
+    created_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was created"})  # fmt: skip
     delegated_managed_identity_resource_id: Optional[str] = field(default=None, metadata={'description': 'Id of the delegated managed identity resource'})  # fmt: skip
     description: Optional[str] = field(default=None, metadata={"description": "Description of role assignment"})
     principal_id: Optional[str] = field(default=None, metadata={"description": "The principal ID."})
     principal_type: Optional[str] = field(default=None, metadata={'description': 'The principal type of the assigned principal ID.'})  # fmt: skip
     role_definition_id: Optional[str] = field(default=None, metadata={"description": "The role definition ID."})
     scope: Optional[str] = field(default=None, metadata={"description": "The role assignment scope."})
     updated_by: Optional[str] = field(default=None, metadata={"ignore_history": True, 'description': 'Id of the user who updated the assignment'})  # fmt: skip
-    updated_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was updated"})  # fmt: skip
+    updated_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was updated"})  # fmt: skip
 
     def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None:
         # role definition
@@ -259,12 +259,12 @@ class AzureAuthorizationRoleDefinition(MicrosoftResource, BaseRole, PhantomBaseR
     }
     assignable_scopes: Optional[List[str]] = field(default=None, metadata={'description': 'Role definition assignable scopes.'})  # fmt: skip
     created_by: Optional[str] = field(default=None, metadata={'description': 'Id of the user who created the assignment'})  # fmt: skip
-    created_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was created"})  # fmt: skip
+    created_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was created"})  # fmt: skip
     description: Optional[str] = field(default=None, metadata={"description": "The role definition description."})
     azure_role_permissions: Optional[List[AzurePermission]] = field(default=None, metadata={'description': 'Role definition permissions.'})  # fmt: skip
     role_name: Optional[str] = field(default=None, metadata={"description": "The role name."})
     updated_by: Optional[str] = field(default=None, metadata={"ignore_history": True, 'description': 'Id of the user who updated the assignment'})  # fmt: skip
-    updated_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was updated"})  # fmt: skip
+    updated_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was updated"})  # fmt: skip
 
 
 @define(eq=False, slots=False)