XSUP 31502 - Gmail - Update the time parsing during the fetch-inciden…

…ts process (#32431)

* updated get_occurred_date to return the smaller of the header time and internalDate in utc

* base_time to occured

* added more debug logs for the body of the msg

* RN and pack version update

* updated the failing tests

* pre-commit updates

* docker image update

* updated the unit test test_get_occurred_date

* pre-commit updates

* cr updates

Packs/Gmail/Integrations/Gmail/Gmail.py

@@ -281,6 +281,7 @@ def get_occurred_date(email_data: dict) -> Tuple[datetime, bool]:
         Tuple[datetime, bool]: occurred datetime, can be used for incrementing search date
     """
     headers = demisto.get(email_data, 'payload.headers')
+    output = None
     if not headers or not isinstance(headers, list):
         demisto.error(f"couldn't get headers for msg (shouldn't happen): {email_data}")
     else:
@@ -292,18 +293,28 @@ def get_occurred_date(email_data: dict) -> Tuple[datetime, bool]:
                 if val:
                     res = get_date_from_email_header(val)
                     if res:
-                        demisto.debug(f"Using occurred date: {res} from header: {name} value: {val}")
-                        return res, True
+                        output = datetime.fromtimestamp(res.timestamp(), tz=timezone.utc)
+                        demisto.debug(f"The timing from header: {name} value: {val} the result: {res}, the UTC time is {output}")
+                        break
     internalDate = email_data.get('internalDate')
-    demisto.info(f"couldn't extract occurred date from headers trying internalDate: {internalDate}")
+    demisto.info(f"trying internalDate: {internalDate}")
     if internalDate and internalDate != '0':
         # intenalDate timestamp has 13 digits, but epoch-timestamp counts the seconds since Jan 1st 1970
         # (which is currently less than 13 digits) thus a need to cut the timestamp down to size.
         timestamp_len = len(str(int(time.time())))
-        if len(str(internalDate)) > timestamp_len:
+        if len(str(internalDate)) >= timestamp_len:
             internalDate = (str(internalDate)[:timestamp_len])
-        return datetime.fromtimestamp(int(internalDate), tz=timezone.utc), True
-        # we didn't get a date from anywhere
+            internalDate_dt = datetime.fromtimestamp(int(internalDate), tz=timezone.utc)
+            demisto.debug(f"{internalDate=} {internalDate_dt=}")
+            if output and internalDate_dt:
+                # check which time is earlier, return it
+                output = internalDate_dt if internalDate_dt < output else output
+            elif internalDate_dt and not output:
+                output = internalDate_dt
+    if output:
+        demisto.debug(f"The final occurred time is {output}")
+        return output, True
+    # we didn't get a date from anywhere
     demisto.info("Failed finding date from internal or headers. Using 'datetime.now()'")
     return datetime.now(tz=timezone.utc), False
 
@@ -340,11 +351,7 @@ def get_email_context(email_data, mailbox):
     body = demisto.get(email_data, 'payload.body.data')
     body = body.encode('ascii') if body is not None else ''
     parsed_body = base64.urlsafe_b64decode(body)
-    base_time = email_data.get('internalDate')
-    if not base_time or not get_date_from_email_header(base_time):
-        # we have an invalid date. use the occurred in rfc 2822
-        demisto.debug(f'Using Date base time from occurred: {occurred} instead of date header: [{base_time}]')
-        base_time = format_datetime(occurred)
+    demisto.debug(f"get_email_context {body=} {parsed_body=}")
 
     context_gmail = {
         'Type': 'Gmail',
@@ -367,7 +374,7 @@ def get_email_context(email_data, mailbox):
         # only for incident
         'Cc': headers.get('cc', []),
         'Bcc': headers.get('bcc', []),
-        'Date': base_time,
+        'Date': format_datetime(occurred),
         'Html': None,
     }
 
@@ -387,7 +394,7 @@ def get_email_context(email_data, mailbox):
 
         'CC': headers.get('cc', []),
         'BCC': headers.get('bcc', []),
-        'Date': base_time,
+        'Date': format_datetime(occurred),
         'Body/HTML': None,
     }
 
@@ -396,10 +403,12 @@ def get_email_context(email_data, mailbox):
         context_gmail['Body'] = html_to_text(context_gmail['Body'])
         context_email['Body/HTML'] = context_gmail['Html']
         context_email['Body/Text'] = context_gmail['Body']
+        demisto.debug(f"In text/html {context_gmail['Body']=}")
 
     if 'multipart' in context_gmail['Format']:  # type: ignore
         context_gmail['Body'], context_gmail['Html'], context_gmail['Attachments'] = parse_mail_parts(
             email_data.get('payload', {}).get('parts', []))
+        demisto.debug(f"In multipart {context_gmail['Body']=}")
         context_gmail['Attachment Names'] = ', '.join(
             [attachment['Name'] for attachment in context_gmail['Attachments']])  # type: ignore
         context_email['Body/Text'], context_email['Body/HTML'], context_email['Attachments'] = parse_mail_parts(

Packs/Gmail/Integrations/Gmail/Gmail.yml

@@ -1715,7 +1715,7 @@ script:
     - contextPath: Gmail.ForwardingAddress.verificationStatus
       description: Indicates whether this address has been verified and is usable for forwarding.
       type: String
-  dockerimage: demisto/google-api-py3:1.0.0.82944
+  dockerimage: demisto/google-api-py3:1.0.0.86786
   isfetch: true
   runonce: false
   script: '-'