[Gmail] filter fields bug (#27403)

* Fix for the fields argument that didn't work in the gmail-search and gmail-search-all-mailboxes commands

* add RN

* commit

* update docker and correction comments

* fix UT

* update docker

* correction comment

* commit

Packs/Gmail/Integrations/Gmail/Gmail.py

@@ -191,6 +191,41 @@ def parse_mail_parts(parts):
     return body, html, attachments
 
 
+def format_fields_argument(fields: list[str]) -> list[str] | None:
+    """
+    Checks if the filter fields are valid, if so returns the valid fields,
+    otherwise returns `None`, when given an empty list returns `None`.
+    """
+    all_valid_fields = (
+        "Type",
+        "Mailbox",
+        "ThreadId",
+        "Labels",
+        "Headers",
+        "Attachments",
+        "RawData",
+        "Format",
+        "Subject",
+        "From",
+        "To",
+        "Body",
+        "Cc",
+        "Bcc",
+        "Date",
+        "Html",
+        "Attachment Names",
+    )
+    lower_filter_fields = {field.lower() for field in fields}
+    if valid_fields := [field for field in all_valid_fields if field.lower() in lower_filter_fields]:
+        valid_fields.append('ID')
+        return valid_fields
+    return None
+
+
+def filter_by_fields(full_mail: dict[str, Any], filter_fields: list[str]) -> dict:
+    return {field: full_mail.get(field) for field in filter_fields}
+
+
 def parse_privileges(raw_privileges):
     privileges = []
     for p in raw_privileges:
@@ -442,11 +477,13 @@ def mailboxes_to_entry(mailboxes: list[dict]) -> list[CommandResults]:
     return command_results
 
 
-def emails_to_entry(title, raw_emails, format_data, mailbox):
+def emails_to_entry(title, raw_emails, format_data, mailbox, fields: list[str] | None = None):
     gmail_emails = []
     emails = []
     for email_data in raw_emails:
         context_gmail, _, context_email, occurred, occurred_is_valid = get_email_context(email_data, mailbox)
+        if fields:
+            context_gmail = filter_by_fields(context_gmail, fields)
         gmail_emails.append(context_gmail)
         emails.append(context_email)
 
@@ -797,7 +834,7 @@ def scheduled_commands_for_more_users(accounts: list, next_page_token: str) -> L
         args.update({'list_accounts': batch})
         command_results.append(
             CommandResults(
-                readable_output='Serching mailboxes, please wait...',
+                readable_output='Searching mailboxes, please wait...',
                 scheduled_command=ScheduledCommand(
                     command='gmail-search-all-mailboxes',
                     next_run_in_seconds=10,
@@ -1350,7 +1387,7 @@ def search_command(mailbox: str = None, only_return_account_names: bool = False)
     _in = args.get('in', '')
 
     query = args.get('query', '')
-    fields = args.get('fields')
+    fields = format_fields_argument(argToList(args.get('fields')))
     label_ids = [lbl for lbl in args.get('labels-ids', '').split(',') if lbl != '']
     max_results = int(args.get('max-results', 100))
     page_token = args.get('page-token')
@@ -1379,7 +1416,7 @@ def search_command(mailbox: str = None, only_return_account_names: bool = False)
             return {'Mailbox': mailbox, 'q': q}
         return None
     if mails:
-        res = emails_to_entry(f'Search in {mailbox}:\nquery: "{q}"', mails, 'full', mailbox)
+        res = emails_to_entry(f'Search in {mailbox}:\nquery: "{q}"', mails, 'full', mailbox, fields)
         return res
     return None
 
@@ -1405,7 +1442,6 @@ def search(user_id, subject='', _from='', to='', before='', after='', filename='
         'userId': user_id,
         'q': q,
         'maxResults': max_results,
-        'fields': fields,
         'labelIds': label_ids,
         'pageToken': page_token,
         'includeSpamTrash': include_spam_trash,

Packs/Gmail/Integrations/Gmail/Gmail.yml

@@ -370,8 +370,10 @@ script:
       required: false
       secret: false
     - default: false
-      description: Enables partial responses to be retrieved, separated by commas. For more information, see https://developers.google.com/gdata/docs/2.0/basics#PartialResponse.
-      isArray: false
+      description: 'Enables partial responses to be retrieved, separated by commas. Valid fields are only from the following list: 
+        Type, Mailbox, ThreadId, Labels, Headers, Attachments, RawData, Format, Subject, From, To, Body, Cc, Bcc,
+        Date, Html, Attachment Names'
+      isArray: true
       name: fields
       required: false
       secret: false
@@ -542,8 +544,10 @@ script:
       required: false
       secret: false
     - default: false
-      description: Enables partial responses to be retrieved in a comma-separated list. For more information, see https://developers.google.com/gdata/docs/2.0/basics#PartialResponse.
-      isArray: false
+      description: 'Enables partial responses to be retrieved, separated by commas. Valid fields are only from the following list: 
+        Type, Mailbox, ThreadId, Labels, Headers, Attachments, RawData, Format, Subject, From, To, Body, Cc, Bcc,
+        Date, Html, Attachment Names'
+      isArray: true
       name: fields
       required: false
       secret: false
@@ -2316,7 +2320,7 @@ script:
     - contextPath: Gmail.ForwardingAddress.verificationStatus
       description: Indicates whether this address has been verified and is usable for forwarding.
       type: String
-  dockerimage: demisto/google-api-py3:1.0.0.61016
+  dockerimage: demisto/google-api-py3:1.0.0.63203
   isfetch: true
   longRunning: false
   longRunningPort: false

Packs/Gmail/Integrations/Gmail/Gmail_test.py

@@ -765,3 +765,82 @@ def test_parse_date_isoformat_server():
     date = parse_date_isoformat_server('2017-10-24T14:13:20Z')
     assert date == datetime.datetime(2017, 10, 24, 14, 13, 20, tzinfo=datetime.timezone.utc)
     assert str(date) == '2017-10-24 14:13:20+00:00'
+
+
+@pytest.mark.parametrize(
+    "fields, expected_result",
+    [
+        (None, None),
+        ("test", None),
+        ("test,test", None),
+        ("id", None),
+        ("subject", ["Subject", "ID"]),
+        ("subject,html", ["Subject", "Html", "ID"]),
+    ],
+)
+def test_format_fields_argument(fields: str | None, expected_result: list[str] | None):
+    """
+    Given:
+        - A string or None in fields argument.
+
+    When:
+        - `format_fields_argument` is called with the fields.
+
+    Then:
+        Ensure:
+        - When given None returns None
+        - When an invalid field is given, None is returned
+        - When valid fields are given the list returned contains the given fields
+        - When valid fields are given, the ID field will always be included in the list
+    """
+    from Gmail import format_fields_argument
+    from CommonServerPython import argToList
+    assert format_fields_argument(argToList(fields)) == expected_result
+
+
+@pytest.mark.parametrize(
+    "full_mail, filter_fields, expected_result",
+    [
+        (
+            {
+                "Type": "Gmail",
+                "Mailbox": "test",
+                "ID": "id",
+                "ThreadId": "test",
+                "Labels": "test",
+                "Headers": "test",
+                "Attachments": "test",
+                "RawData": "test",
+                "Format": "test",
+                "Subject": "test",
+                "From": "test",
+                "To": "test",
+                "Body": "test",
+                "Cc": "test",
+                "Bcc": "test",
+                "Date": "test",
+                "Html": "test",
+            },
+            ["ID", "Labels", "From"],
+            {
+                "ID": "id",
+                "Labels": "test",
+                "From": "test",
+            },
+        )
+    ],
+)
+def test_filter_by_fields(
+    full_mail: dict[str, str], filter_fields: list[str], expected_result: dict[str, str]
+):
+    """
+    Given:
+        - A full mail object, a list of filter fields.
+    When:
+        - `filter_by_fields` is called.
+    Then:
+        - Ensure the email is filtered by the fields that given.
+    """
+    from Gmail import filter_by_fields
+
+    assert filter_by_fields(full_mail, filter_fields) == expected_result

Packs/Gmail/Integrations/Gmail/README.md

@@ -765,7 +765,7 @@ Searches for Gmail records for a specific Google user.
 | user-id | The user's email address. The "me" special value can be used to indicate the authenticated user. | Required | 
 | query | Returns messages matching the specified query. Supports the same query format as the Gmail search box. For example, "from:someuser@example.com rfc822msgid: is:unread". For more syntax information see "https://support.google.com/mail/answer/7190?hl=en". | Optional | 
 | max-results | The maximum number of results to return. Default is 100. Maximum is 500. Can be 1 to 500, inclusive. Default is 100. | Optional | 
-| fields | Enables partial responses to be retrieved, separated by commas. For more information, see <https://developers.google.com/gdata/docs/2.0/basics#PartialResponse>. | Optional | 
+| fields | Enables partial responses to be retrieved, separated by commas. Valid fields are only from the following list: Type, Mailbox, ThreadId, Labels, Headers, Attachments, RawData, Format, Subject, From, To, Body, Cc, Bcc, Date, Html, Attachment Names. | Optional | 
 | labels-ids | Returns messages with labels that match all of the specified label IDs in a comma-separated list. | Optional | 
 | page-token | Page token to retrieve a specific page of results in the list. | Optional | 
 | include-spam-trash | Include messages from SPAM and TRASH in the results. (Default: false). Possible values are: False, True. Default is False. | Optional | 
@@ -1351,7 +1351,7 @@ Inspecting these messages should allow you to determine what percent the search
 | --- | --- | --- |
 | query | Returns messages matching the specified query. Supports the same query format as the Gmail search box. For example, "from:someuser@example.com rfc822msgid: is:unread". For more syntax information,see "https://support.google.com/mail/answer/7190?hl=en". | Optional | 
 | max-results | The maximum number of results to return. Default is 100. Maximum is 500. Can be 1 to 500, inclusive. Default is 100. | Optional | 
-| fields | Enables partial responses to be retrieved in a comma-separated list. For more information, see <https://developers.google.com/gdata/docs/2.0/basics#PartialResponse>. | Optional | 
+| fields | Enables partial responses to be retrieved, separated by commas. Valid fields are only from the following list: Type, Mailbox, ThreadId, Labels, Headers, Attachments, RawData, Format, Subject, From, To, Body, Cc, Bcc, Date, Html, Attachment Names. | Optional | 
 | labels-ids | Returns messages with labels that match all of the specified label IDs in a comma-separated list. | Optional | 
 | page-token | Page token to retrieve a specific page of results in the list. | Optional | 
 | include-spam-trash | Includes messages from SPAM and TRASH in the results. (Default: false). Possible values are: False, True. Default is False. | Optional | 

Packs/Gmail/ReleaseNotes/1_3_10.md

@@ -0,0 +1,6 @@
+
+#### Integrations
+
+##### Gmail
+- Updated the Docker image to: *demisto/google-api-py3:1.0.0.63203*.
+- Fixed an issue where the *fields* argument in the ***gmail-search*** and ***gmail-search-all-mailboxes*** commands did not work.

Packs/Gmail/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Gmail",
     "description": "Gmail API and user management (This integration replaces the Gmail functionality in the GoogleApps API and G Suite integration).",
     "support": "xsoar",
-    "currentVersion": "1.3.9",
+    "currentVersion": "1.3.10",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",