Fix test-upload merge general release notes (#30041)

Tests/Marketplace/Tests/marketplace_services_test.py

@@ -1526,7 +1526,44 @@ def test_create_filtered_changelog_entry_no_related_modifications(self, dummy_pa
 <~XSIAM>
 - **Field Name 1**
 - **Field Name 2**
-</~XSIAM>''', 'xsoar', 'Changes are not relevant for XSOAR marketplace.')
+</~XSIAM>''', 'xsoar', 'Changes are not relevant for XSOAR marketplace.'),
+        (  # Case 11
+            '''
+## PackName
+- General pack change
+
+#### Integrations
+##### Integration Display Name
+<~XSIAM>
+- Fixed an issue
+</~XSIAM>''', 'xsoar', '## PackName\n- General pack change'),
+        (  # Case 12
+            '''
+## PackName
+- General pack change''', 'xsoar', '## PackName\n- General pack change'),
+        (  # Case 13
+            '''
+## PackName
+- General pack change
+
+#### Integrations
+##### Integration Display Name
+<~XSOAR>
+- Fixed an issue
+</~XSOAR>''', 'xsoar',
+            '## PackName\n- General pack change\n#### Integrations\n##### Integration Display Name\n- Fixed an issue'),
+        (  # Case 14
+            '''
+## PackName
+<~XSOAR>
+- General pack change
+<~XSOAR>''', 'xsoar', '## PackName\n- General pack change'),
+        (  # Case 15
+            '''
+## PackName
+<~XSIAM>
+- General pack change
+</~XSIAM>''', 'xsoar', 'Changes are not relevant for XSOAR marketplace.'),
     ])
     def test_create_filtered_changelog_entry_by_mp_tags(self, dummy_pack: Pack, release_notes, upload_marketplace,
                                                         expected_result):
@@ -1543,15 +1580,32 @@ def test_create_filtered_changelog_entry_by_mp_tags(self, dummy_pack: Pack, rele
                  Case 8: Test for new entities with the 'New' in display name for the same marketplace.
                  Case 9: Same as case 8 but for the other marketplace.
                  Case 10: Eentities like incident fields in RN have wrapping tags in their entries and not relevant for MP.
+                 Case 11: General pack notes along with XSIAM only changes for xsoar marketplace.
+                 Case 12: General pack release notes only.
+                 Case 13: General pack release notes along with XSOAR only changes for xsoar marketplace.
+                 Case 14: General pack release notes that are wrapped in XSOAR tags for xsoar marketplace.
+                 Case 15: General pack release notes that are wrapped in XSIAM tags for xsoar marketplace.
            When:
                - Creating changelog entry and filtering the entries by the tags.
            Then:
                - Cases 1-5: Ensure the RN are filtered correctly including the headers / display names if needed.
                - Cases 6-7: Ensure just the tags are removed from RN and not entries.
+               - Cases 11-15: Ensure handling of general pack release notes is the same as other entities.
+
         """
         version_display_name = "1.2.3"
         build_number = "5555"
         id_set = {
+            "Packs": [
+                {
+                    'id':
+                        {
+                            "name": "PackName",
+                            "display_name": "PackName",
+                            "marketplaces": []
+                        }
+                }
+            ],
             "integrations": [
                 {
                     'id':

Tests/Marketplace/marketplace_constants.py

@@ -432,7 +432,8 @@ class Changelog:
     'Triggers Recommendations': 'Triggers',
     'Wizards': 'Wizards',
     'XDRC Templates': 'XDRCTemplates',
-    'Layout Rules': 'LayoutRules'
+    'Layout Rules': 'LayoutRules',
+    'Packs': 'Packs'
 }
 
 # the format is defined in issue #19786, may change in the future

Tests/Marketplace/marketplace_services.py

@@ -1683,7 +1683,7 @@ def filter_changelog_entries(self, changelog_entry: dict, version: str, marketpl
             (bool) Whether the pack is not updated because the entries are not relevant to the current marketplace.
         """
         logging.debug(f"Starting to filter changelog entries by the entities that are given from id-set for pack "
-                      f"{self._pack_name}")
+                      f"{self._pack_name} and marketplace {marketplace}")
 
         release_notes = self.filter_release_notes_by_tags(changelog_entry.get(Changelog.RELEASE_NOTES), marketplace)
 
@@ -1792,7 +1792,7 @@ def release_notes_dont_contain_entities_sections(release_notes_str, release_note
             release_notes_dict (dict): The release notes in dict object.
 
         Returns:
-            (bool) Whther the dict contains the RN entries by the entities types.
+            (bool) Whether the dict contains the RN entries by the entities types.
         """
         return release_notes_str and not release_notes_dict
 
@@ -4207,7 +4207,6 @@ def is_content_item_in_id_set(display_name: str, rn_header: str, id_set: dict, m
         return True
 
     for id_set_entity in id_set[RN_HEADER_TO_ID_SET_KEYS[rn_header]]:
-
         if list(id_set_entity.values())[0]['display_name'] == display_name:
             return True
 

Tests/scripts/infrastructure_tests/release_notes_generator_test.py

@@ -655,3 +655,65 @@ def test_merge_rns_with_gerneral_announcment(self, Pack_name, versions_ls, expec
         rn_block, latest_version = merge_version_blocks(pack_versions_dict)
         assert latest_version == expected_version
         assert rn_block == expected_results
+
+    @pytest.mark.parametrize('pack_rns, expected_rns, expected_version', [
+        pytest.param({
+            "1.0.1": "## SomePack\n"
+                     "\n"
+                     "- Added some stuff.",
+            "1.0.2": "## SomePack\n"
+                     "\n"
+                     "- Added some other stuff."
+        },
+            "## SomePack\n"
+            "- Added some stuff.\n"
+            "- Added some other stuff.",
+            "1.0.2", id="Merge general notes together"),
+        pytest.param({
+            "1.0.1": "#### Integrations\n"
+                     "##### Some Integration\n"
+                     "- Some stuff.",
+            "1.0.2": "## SomePack\n"
+                     "\n"
+                     "- Added some other stuff."
+        },
+            "## SomePack\n"
+            "- Added some other stuff.\n"
+            "#### Integrations\n"
+            "##### Some Integration\n"
+            "- Some stuff.",
+            "1.0.2", id="Merge general notes with entity RNs"),
+        pytest.param({
+            "1.0.1": "## SomePack\n"
+                     "\n"
+                     "- Added some stuff.\n"
+                     "#### Integrations\n"
+                     "##### Some Integration\n"
+                     "- Some stuff.",
+            "1.0.2": "## SomePack\n"
+                     "\n"
+                     "- Added some other stuff."
+        },
+            "## SomePack\n"
+            "- Added some stuff.\n"
+            "- Added some other stuff.\n"
+            "#### Integrations\n"
+            "##### Some Integration\n"
+            "- Some stuff.",
+            "1.0.2", id="Merge combined notes with general notes"),
+    ])
+    def test_merge_rns_with_general_notes(self, pack_rns: dict, expected_rns: str, expected_version: str):
+        """
+        Given: Two consecutive versions of RNs.
+            - Case 1: Both containing general pack notes.
+            - Case 2: One contains a content entity change and one contains general pack notes.
+            - Case 3: One is combined (content entity change & general note) and one contains general pack notes.
+        When: Using merge_version_blocks function.
+        Then: Ensure that the merge was done correctly:
+            - General notes were merged and are on top.
+            - All other notes are also present.
+            - The latest version is as expected.
+        """
+        merged_rn_block, latest_version = merge_version_blocks(pack_rns)
+        assert merged_rn_block == expected_rns
+        assert latest_version == expected_version

Utils/release_notes_generator.py

@@ -6,6 +6,7 @@
 import argparse
 from datetime import datetime
 import logging
+import copy
 
 from packaging.version import Version
 import requests
@@ -22,6 +23,7 @@
 ENTITY_TYPE_SECTION_REGEX = re.compile(r'^#### ([\w ]+)$\n([\w\W]*?)(?=^#### )|^#### ([\w ]+)$\n([\w\W]*)', re.M)
 ENTITY_SECTION_REGEX = re.compile(r'^##### (.+)$\n([\w\W]*?)(?=^##### )|^##### (.+)$\n([\w\W]*)|'
                                   r'^- \*\*(.+)\*\*$\n([\w\W]*)', re.M)
+PACK_GENERAL_NOTES_REGEX = re.compile(r'^## (.+)$\n([\w\W]*?)(?=^#### )|^## ([\w ]+)$\n([\w\W]*)', re.M)
 
 LAYOUT_TYPE_TO_NAME = {
     "details": "Summary",
@@ -104,6 +106,9 @@ def construct_entities_block(entities_data: dict) -> str:
     Args:
         entities_data (dict): dictionary of the form:
             {
+                Packs: {
+                    PackName: <description>
+                }
                 Integrations: {
                     Integration1: <description>,
                     Integration2:<description>,
@@ -120,6 +125,13 @@ def construct_entities_block(entities_data: dict) -> str:
 
     """
     release_notes = ''
+
+    # Keep general notes on top.
+    general_notes = entities_data.pop('Packs', None)
+    if general_notes:
+        pack_name, general_comments = list(general_notes.items())[0]
+        release_notes += f'## {pack_name}\n{general_comments}'
+
     for entity_type, entities_description in sorted(entities_data.items()):
         pretty_entity_type = re.sub(r'([a-z])([A-Z])', r'\1 \2', entity_type)
         release_notes += f'#### {pretty_entity_type}\n'
@@ -154,8 +166,8 @@ def get_pack_entities(pack_path):
         if match:
             entity_type = match.group(1)
         else:
-            # should not get here
-            entity_type = 'Extras'
+            # General pack comment
+            entity_type = 'Packs'
 
         name, description = get_new_entity_record(entity_path)
         entities_data.setdefault(entity_type, {})[name] = description
@@ -306,6 +318,37 @@ def aggregate_release_notes(pack_name: str, pack_versions_dict: dict, pack_metad
             f'{pack_release_notes}')
 
 
+def append_commment_to_data(entities_data, entity_type, entity_name, entity_comment):
+    """Append release note comment to the right place in merged dict.
+
+    Note: This function copies the provided dictionary and creates a different updated copy.
+
+    Args:
+        entities_data: A dictionary of the current release notes merge.
+        entity_type: The type of entity the release notes are concerning. (Integration / Playbook / etc).
+        entity_name: The name of the content entity.
+        entity_comment: The release note comment to append.
+
+    Returns:
+        dict: the updated dictionary.
+    """
+    entities_data_new = copy.deepcopy(entities_data)
+    # release notes of the entity
+    if entity_name in entities_data_new.get(entity_type, {}):
+        exists_comment = entities_data_new.get(entity_type, {}).get(entity_name)
+        if entity_comment and entity_name != '[special_msg]':
+            if not exists_comment.startswith('- '):
+                logging.debug(f'Adding missing "-" to entity comment: {exists_comment}')
+                entities_data_new[entity_type][entity_name] = f'- {exists_comment}'
+            if not entity_comment.strip().startswith('- '):
+                logging.debug(f'Adding missing "-" to entity comment: {entity_comment}')
+                entity_comment = f'- {entity_comment.strip()}'
+        entities_data_new[entity_type][entity_name] += f'{entity_comment.strip()}\n'
+    else:
+        entities_data_new[entity_type][entity_name] = f'{entity_comment.strip()}\n'
+    return entities_data_new
+
+
 def merge_version_blocks(pack_versions_dict: dict, return_str: bool = True) -> tuple[str | dict, str]:
     """
     merge several pack release note versions into a single block.
@@ -324,6 +367,20 @@ def merge_version_blocks(pack_versions_dict: dict, return_str: bool = True) -> t
     for pack_version, version_release_notes in sorted(pack_versions_dict.items(),
                                                       key=lambda pack_item: Version(pack_item[0])):
         latest_version = pack_version
+
+        # Handle general pack notes.
+        general_note_sections = PACK_GENERAL_NOTES_REGEX.findall(version_release_notes)
+        if general_note_sections:
+            # Assume only one general notes section at the start of the file.
+            general_note_components = general_note_sections[0]
+            entity_type = 'Packs'
+            entities_data.setdefault(entity_type, {})
+            # Extract Pack name
+            entity_name = general_note_components[0] or general_note_components[2]
+            # General release notes of the pack
+            entity_comment = general_note_components[1] or general_note_components[3]
+            entities_data = append_commment_to_data(entities_data, entity_type, entity_name, entity_comment)
+
         version_release_notes = version_release_notes.strip()
         # extract release notes sections by content types (all playbooks, all scripts, etc...)
         # assuming all entity titles start with level 4 header ("####") and then a list of all comments
@@ -345,18 +402,7 @@ def merge_version_blocks(pack_versions_dict: dict, return_str: bool = True) -> t
                 entity_name = entity_name.replace('__', '')
                 # release notes of the entity
                 entity_comment = entity[1] or entity[3] or entity[5]
-                if entity_name in entities_data[entity_type]:
-                    exists_comment = entities_data[entity_type][entity_name]
-                    if entity_comment and entity_name != '[special_msg]':
-                        if not exists_comment.startswith('- '):
-                            logging.debug(f'Adding missing "-" to entity comment: {exists_comment}')
-                            entities_data[entity_type][entity_name] = f'- {exists_comment}'
-                        if not entity_comment.strip().startswith('- '):
-                            logging.debug(f'Adding missing "-" to entity comment: {entity_comment}')
-                            entity_comment = f'- {entity_comment.strip()}'
-                    entities_data[entity_type][entity_name] += f'{entity_comment.strip()}\n'
-                else:
-                    entities_data[entity_type][entity_name] = f'{entity_comment.strip()}\n'
+                entities_data = append_commment_to_data(entities_data, entity_type, entity_name, entity_comment)
 
     pack_release_notes = construct_entities_block(entities_data).strip() if return_str else entities_data
 

Utils/test_upload_flow/create_test_branch.py

@@ -294,8 +294,8 @@ def do_changes_on_branch(packs_path: Path):
     # Case 4: Verify new version - ZeroFox
     enhance_release_notes(packs_path / 'ZeroFox')
 
-    # Case 5: Verify modified existing release notes - BPA
-    update_existing_release_notes(packs_path / 'BPA')
+    # Case 5: Verify modified existing release notes - Box
+    update_existing_release_notes(packs_path / 'Box')
 
     # Case 6: Verify pack is set to hidden - Microsoft365Defender
     set_pack_hidden(packs_path / 'Microsoft365Defender')

Utils/test_upload_flow/verify_bucket.py

@@ -32,7 +32,7 @@
 
 
 def read_json(path):
-    with open(path, 'r') as file:
+    with open(path) as file:
         return json.load(file)
 
 
@@ -173,7 +173,7 @@ def get_pack_readme(self, pack_id):
         Returns the pack README file
         """
         item_path = os.path.join(self.extracting_destination, pack_id, 'README.md')
-        with open(item_path, 'r') as f:
+        with open(item_path) as f:
             return f.read()
 
 
@@ -335,9 +335,9 @@ def run_validations(self):
         expected_rn = 'testing adding new RN'
         self.verify_new_version('ZeroFox', expected_rn)
 
-        # Case 5: Verify modified existing release notes - BPA
+        # Case 5: Verify modified existing release notes - Box
         expected_rn = 'testing modifying existing RN'
-        self.verify_rn('BPA', expected_rn)
+        self.verify_rn('Box', expected_rn)
 
         # Case 6: Verify pack is set to hidden - Microsoft365Defender
         self.verify_hidden('Microsoft365Defender')