feat: add parameter value origin field to parameters (#1470)

* feat: add parameter value origin field to parameters

Signed-off-by: Alejandro Jose Leiva Palomo <alejandro.leiva.palomo@ibm.com>

* fix: remove wrong added field from oscal model

Signed-off-by: Alejandro Jose Leiva Palomo <alejandro.leiva.palomo@ibm.com>

* fix: add param_value_origin to props and add validations

Signed-off-by: Alejandro Jose Leiva Palomo <alejandro.leiva.palomo@ibm.com>

* fix: correct ci

Signed-off-by: Alejandro Jose Leiva Palomo <alejandro.leiva.palomo@ibm.com>

* fix: correct param value origin cycle

Signed-off-by: Alejandro Jose Leiva Palomo <alejandro.leiva.palomo@ibm.com>

* fix: correct profile-param-value-origin flow

Signed-off-by: Alejandro Jose Leiva Palomo <alejandro.leiva.palomo@ibm.com>

* fix: adding final corrections and test for inherited param-value-origin

Signed-off-by: Alejandro Jose Leiva Palomo <alejandro.leiva.palomo@ibm.com>

* fix: correct formating

Signed-off-by: Alejandro Jose Leiva Palomo <alejandro.leiva.palomo@ibm.com>

* fix: add step to ignore param-value-origin if no replacement was done in profile-param-value-origin

Signed-off-by: Alejandro Jose Leiva Palomo <alejandro.leiva.palomo@ibm.com>

* fix: correct code format

Signed-off-by: Alejandro Jose Leiva Palomo <alejandro.leiva.palomo@ibm.com>

* fix: correct tests

Signed-off-by: Alejandro Jose Leiva Palomo <alejandro.leiva.palomo@ibm.com>

* fix: use replace me placeholder instead of literal text

Signed-off-by: Alejandro Jose Leiva Palomo <alejandro.leiva.palomo@ibm.com>

* fix: use replace me tag in default value for param-value-origin

Signed-off-by: Alejandro Jose Leiva Palomo <alejandro.leiva.palomo@ibm.com>

* fix: correct code format

Signed-off-by: Alejandro Jose Leiva Palomo <alejandro.leiva.palomo@ibm.com>

---------

Signed-off-by: Alejandro Jose Leiva Palomo <alejandro.leiva.palomo@ibm.com>

tests/data/json/simple_test_profile.json

@@ -36,6 +36,10 @@
               "name": "display-name",
               "ns": "https://oscal-compass.github.io/compliance-trestle/schemas/oscal",
               "value": "Pretty ac-1 prm 1"
+            },
+            {
+              "name": "param-value-origin",
+              "value": "comes from xyz policy"
             }
           ],
           "label": "label from profile",

tests/data/json/test_profile_b.json

@@ -48,6 +48,10 @@
               {
                 "name": "set_param_prof_b_prop",
                 "value": "set param prof b prop value"
+              },
+              {
+                "name": "param-value-origin",
+                "value": "comes from xyz policiy"
               }
             ]
         },

tests/trestle/core/commands/author/profile_test.py

@@ -1300,3 +1300,151 @@ def test_profile_generate_assesment_objectives(tmp_trestle_dir: pathlib.Path, mo
     monkeypatch.setattr(sys, 'argv', test_args)
 
     assert Trestle().run() == 0
+
+
+def test_profile_generate_assemble_param_value_origin(tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None:
+    """Test the profile markdown generator."""
+    _, assembled_prof_dir, _, markdown_path = setup_profile_generate(tmp_trestle_dir, 'simple_test_profile.json')
+    yaml_header_path = test_utils.YAML_TEST_DATA_PATH / 'good_simple.yaml'
+
+    # convert resolved profile catalog to markdown then assemble it after adding an item to a control
+    # generate, edit, assemble
+    test_args = f'trestle author profile-generate -n {prof_name} -o {md_name} -rs NeededExtra'.split(  # noqa E501
+    )
+    test_args.extend(['-y', str(yaml_header_path)])
+    test_args.extend(['-s', all_sections_str])
+    monkeypatch.setattr(sys, 'argv', test_args)
+
+    assert Trestle().run() == 0
+
+    md_path = markdown_path / 'ac' / 'ac-1.md'
+    assert md_path.exists()
+    md_api = MarkdownAPI()
+    header, tree = md_api.processor.process_markdown(md_path)
+
+    assert header
+    assert header[const.SET_PARAMS_TAG]['ac-1_prm_1'][const.PROFILE_PARAM_VALUE_ORIGIN] == 'comes from xyz policy'
+    header[const.SET_PARAMS_TAG]['ac-1_prm_1'][const.PROFILE_PARAM_VALUE_ORIGIN] = 'Needed to change param value origin'
+
+    md_api.write_markdown_with_header(md_path, header, tree.content.raw_text)
+
+    # assemble based on set_parameters_flag
+    test_args = f'trestle author profile-assemble -n {prof_name} -m {md_name} -o {assembled_prof_name}'.split()
+    test_args.append('-sp')
+    assembled_prof_dir.mkdir()
+    monkeypatch.setattr(sys, 'argv', test_args)
+    assert Trestle().run() == 0
+
+    profile, _ = ModelUtils.load_model_for_class(tmp_trestle_dir, 'my_assembled_prof',
+                                                 prof.Profile, FileContentType.JSON)
+
+    # grabs first parameter in line and test out the value
+    assert profile.modify.set_parameters[0].props[1].value == 'Needed to change param value origin'
+
+    profile.modify.set_parameters[0].props[1].value = 'this is a change test'
+
+    ModelUtils.save_top_level_model(profile, tmp_trestle_dir, 'my_assembled_prof', FileContentType.JSON)
+
+    # convert resolved profile catalog to markdown then assemble it after adding an item to a control
+    # generate, edit, assemble
+    test_args = f'trestle author profile-generate -n {assembled_prof_name} -o {md_name} -rs NeededExtra --force-overwrite'.split(  # noqa E501
+    )
+    test_args.extend(['-y', str(yaml_header_path)])
+    test_args.extend(['-s', all_sections_str])
+    monkeypatch.setattr(sys, 'argv', test_args)
+
+    assert Trestle().run() == 0
+
+    md_path = markdown_path / 'ac' / 'ac-1.md'
+    assert md_path.exists()
+    md_api = MarkdownAPI()
+    header, tree = md_api.processor.process_markdown(md_path)
+
+    assert header
+    header[const.SET_PARAMS_TAG]['ac-1_prm_1'][const.PROFILE_PARAM_VALUE_ORIGIN] = 'now again I need to change origin'
+
+    md_api.write_markdown_with_header(md_path, header, tree.content.raw_text)
+
+    # assemble based on set_parameters_flag
+    test_args = f'trestle author profile-assemble -n {prof_name} -m {md_name} -o {assembled_prof_name} -r'.split()
+    test_args.append('-sp')
+    monkeypatch.setattr(sys, 'argv', test_args)
+    assert Trestle().run() == 0
+
+
+def test_param_value_origin_from_inherited_profile(tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None:
+    """Test the inherited param-value-origin coming from imported profile."""
+    test_utils.setup_for_multi_profile(tmp_trestle_dir, False, True)
+    yaml_header_path = test_utils.YAML_TEST_DATA_PATH / 'good_simple.yaml'
+
+    # convert resolved profile catalog to markdown then assemble it after adding an item to a control
+    # generate, edit, assemble
+    test_args = f'trestle author profile-generate -n test_profile_a -o {md_name} -rs NeededExtra'.split(  # noqa E501
+    )
+    test_args.extend(['-y', str(yaml_header_path)])
+    test_args.extend(['-s', all_sections_str])
+    monkeypatch.setattr(sys, 'argv', test_args)
+
+    assert Trestle().run() == 0
+
+    # assemble based on set_parameters_flag
+    test_args = f'trestle author profile-assemble -n test_profile_a -m {md_name} -o {assembled_prof_name}'.split()
+    test_args.append('-sp')
+    assembled_prof_dir = tmp_trestle_dir / 'profiles/my_assembled_prof'
+    assembled_prof_dir.mkdir()
+    monkeypatch.setattr(sys, 'argv', test_args)
+    assert Trestle().run() == 0
+
+    profile, _ = ModelUtils.load_model_for_class(tmp_trestle_dir, 'my_assembled_prof',
+                                                 prof.Profile, FileContentType.JSON)
+
+    # grabs parameter ac-3.3_prm_1 and verify param-value-origin wasn´t added as
+    # profile-param-value-origin wasn´t modified
+    assert profile.modify.set_parameters[18].props is None
+
+    md_path = tmp_trestle_dir / 'my_md' / 'ac' / 'ac-3.3.md'
+
+    assert md_path.exists()
+    md_api = MarkdownAPI()
+    header, tree = md_api.processor.process_markdown(md_path)
+
+    assert header
+    assert header[const.SET_PARAMS_TAG]['ac-3.3_prm_1'][const.PROFILE_PARAM_VALUE_ORIGIN
+                                                        ] == const.REPLACE_ME_PLACEHOLDER
+    header[const.SET_PARAMS_TAG]['ac-3.3_prm_1'][const.PROFILE_PARAM_VALUE_ORIGIN
+                                                 ] = 'Needed to change param value origin'
+    md_api.write_markdown_with_header(md_path, header, tree.content.raw_text)
+
+    # assemble based on set_parameters_flag
+    test_args = f'trestle author profile-assemble -n test_profile_a -m {md_name} -o {assembled_prof_name}'.split()
+    test_args.append('-sp')
+    monkeypatch.setattr(sys, 'argv', test_args)
+    assert Trestle().run() == 0
+
+    profile, _ = ModelUtils.load_model_for_class(tmp_trestle_dir, 'my_assembled_prof',
+                                                 prof.Profile, FileContentType.JSON)
+
+    # grabs parameter ac-3.3_prm_1 and verify if it was changed correctly
+    assert profile.modify.set_parameters[18].props[0].value == 'Needed to change param value origin'
+    # now change the value in the json file to verify if in the markdown is changed
+    profile.modify.set_parameters[18].props[0].value = 'this is a change test'
+
+    ModelUtils.save_top_level_model(profile, tmp_trestle_dir, 'my_assembled_prof', FileContentType.JSON)
+
+    # convert resolved profile catalog to markdown then assemble it after adding an item to a control
+    # generate, edit, assemble
+    test_args = f'trestle author profile-generate -n {assembled_prof_name} -o {md_name} -rs NeededExtra --force-overwrite'.split(  # noqa E501
+    )
+    test_args.extend(['-y', str(yaml_header_path)])
+    test_args.extend(['-s', all_sections_str])
+    monkeypatch.setattr(sys, 'argv', test_args)
+
+    assert Trestle().run() == 0
+
+    assert md_path.exists()
+    md_api = MarkdownAPI()
+    header, tree = md_api.processor.process_markdown(md_path)
+
+    assert header
+    # verify the change done in json is reflected correctly in the profile-param-value-origin
+    assert header[const.SET_PARAMS_TAG]['ac-3.3_prm_1'][const.PROFILE_PARAM_VALUE_ORIGIN] == 'this is a change test'

trestle/common/const.py

@@ -464,6 +464,10 @@
 
 GUIDELINES = 'guidelines'
 
+PARAM_VALUE_ORIGIN = 'param-value-origin'
+
+PROFILE_PARAM_VALUE_ORIGIN = 'profile-param-value-origin'
+
 LABEL = 'label'
 
 SECTIONS_TAG = TRESTLE_TAG + 'sections'
@@ -533,6 +537,14 @@
   # the parameter value is made up of the values from the other parameters. For parameters
   # that aggregate, profile-values is not applicable.
   #
+  # Property param-value-origin is meant for putting the origin from where that parameter comes from.
+  # In order to be changed in the current profile, profile-param-value-origin property will be displayed with
+  # the placeholder "<REPLACE_ME>" for you to be replaced. If a parameter already has a param-value-origin
+  # coming from an inherited profile, do no change this value, instead use profile-param-value-origin as follows:
+  #
+  #    param-value-origin: DO NOT REPLACE - this is the original value
+  #    profile-param-value-origin: <REPLACE_ME> - replace the new value required HERE
+  #
 """
 
 YAML_RULE_PARAM_VALUES_SSP_COMMENT = """  # You may set new values for rule parameters by adding
@@ -624,3 +636,7 @@
 HELP_LEVERAGED = 'Name of the SSP to be leveraged.'
 
 SATISFIED_STATEMENT_DESCRIPTION = 'Satisfied Statement Description'
+
+ADDED_BY_CONTROL_OWNER = 'Added by control owner'
+
+REPLACE_ME_PLACEHOLDER = '<REPLACE_ME>'

trestle/common/model_utils.py

@@ -631,6 +631,16 @@ def dict_to_parameter(param_dict: Dict[str, Any]) -> common.Parameter:
         if const.AGGREGATES in param_dict:
             # removing aggregates as this is prop just informative in markdown
             param_dict.pop(const.AGGREGATES)
+        param_value_origin = None
+        if const.PARAM_VALUE_ORIGIN in param_dict:
+            param_value_origin = param_dict.pop(const.PARAM_VALUE_ORIGIN)
+            if param_value_origin is not None:
+                props.append(common.Property(name=const.PARAM_VALUE_ORIGIN, value=param_value_origin))
+            else:
+                raise TrestleError(
+                    f'Parameter value origin property for parameter {param_dict["id"]}'
+                    'is None and it should have a value'
+                )
         if const.ALT_IDENTIFIER in param_dict:
             # removing alt-identifier as this is prop just informative in markdown
             param_dict.pop(const.ALT_IDENTIFIER)

trestle/core/catalog/catalog_interface.py

@@ -683,6 +683,14 @@ def _get_display_name_and_ns(param: common.Parameter) -> Tuple[Optional[str], Op
                 return prop.value, ns
         return None, None
 
+    @staticmethod
+    def _get_param_value_origin_and_ns(param: common.Parameter) -> Tuple[Optional[str], Optional[str]]:
+        for prop in as_list(param.props):
+            if prop.name == const.PARAM_VALUE_ORIGIN:
+                ns = str(prop.ns) if prop.ns else None
+                return prop.value, ns
+        return None, None
+
     @staticmethod
     def _prune_controls(md_path: pathlib.Path, written_controls: Set[str]) -> List[str]:
         """Search directory and remove any controls that were not written out."""

trestle/core/catalog/catalog_reader.py

@@ -75,12 +75,25 @@ def read_additional_content(
                     # if profile_values are present, overwrite values with them
                     if const.PROFILE_VALUES in param_dict:
                         if param_dict[const.PROFILE_VALUES] != [] and param_dict[const.PROFILE_VALUES] is not None:
-                            if not write_mode and '<REPLACE_ME>' in param_dict[const.PROFILE_VALUES]:
-                                param_dict[const.PROFILE_VALUES].remove('<REPLACE_ME>')
+                            if not write_mode and const.REPLACE_ME_PLACEHOLDER in param_dict[const.PROFILE_VALUES]:
+                                param_dict[const.PROFILE_VALUES].remove(const.REPLACE_ME_PLACEHOLDER)
                             if param_dict[const.PROFILE_VALUES] != [] and param_dict[const.PROFILE_VALUES] is not None:
                                 param_dict[const.VALUES] = param_dict[const.PROFILE_VALUES]
                         if not write_mode:
                             param_dict.pop(const.PROFILE_VALUES)
+                    # verifies if at control profile edition the param value origin was modified
+                    # through the profile-param-value-origin tag
+                    if const.PROFILE_PARAM_VALUE_ORIGIN in param_dict:
+                        if param_dict[const.PROFILE_PARAM_VALUE_ORIGIN] != const.REPLACE_ME_PLACEHOLDER:
+                            param_dict[const.PARAM_VALUE_ORIGIN] = param_dict[const.PROFILE_PARAM_VALUE_ORIGIN]
+                            param_dict.pop(const.PROFILE_PARAM_VALUE_ORIGIN)
+                        else:
+                            # removes replace me placeholder and profile-param-value-origin as it was not modified
+                            param_dict.pop(const.PROFILE_PARAM_VALUE_ORIGIN)
+                            # validates param-value-origin is in dict to remove it
+                            # because a value wasn´t provided and it shouldn´t be inheriting value from parent
+                            if const.PARAM_VALUE_ORIGIN in param_dict:
+                                param_dict.pop(const.PARAM_VALUE_ORIGIN)
                     final_param_dict[param_id] = param_dict
                     param_sort_map[param_id] = sort_id
         new_alters: List[prof.Alter] = []

trestle/core/catalog/catalog_writer.py

@@ -144,16 +144,30 @@ def _construct_set_parameters_dict(
         for param_id, param_dict in control_param_dict.items():
             # if the param is in the full_param_dict, load its contents first and mark as profile-values
             display_name = ''
+            param_value_origin, _ = CatalogInterface._get_param_value_origin_and_ns(param_dict)
+            prof_param_value_origin = ''
             if param_id in profile_set_param_dict:
                 # get the param from the profile set_param
                 param = profile_set_param_dict[param_id]
                 display_name, _ = CatalogInterface._get_display_name_and_ns(param)
+                prof_param_value_origin, _ = CatalogInterface._get_param_value_origin_and_ns(param)
                 # assign its contents to the dict
                 new_dict = ModelUtils.parameter_to_dict(param, True)
                 if const.VALUES in new_dict:
                     if context.purpose == ContextPurpose.PROFILE:
                         new_dict[const.PROFILE_VALUES] = new_dict[const.VALUES]
                         new_dict.pop(const.VALUES)
+                # validates if parent profile has param-value-origin field
+                if param_value_origin != '' and param_value_origin is not None:
+                    if context.purpose == ContextPurpose.PROFILE:
+                        new_dict[const.PARAM_VALUE_ORIGIN] = param_value_origin
+                # validates if current profile has param-value-origin field and
+                # adds it to prof-param-value-origin
+                if prof_param_value_origin != '' and prof_param_value_origin is not None:
+                    if context.purpose == ContextPurpose.PROFILE:
+                        new_dict[const.PROFILE_PARAM_VALUE_ORIGIN] = prof_param_value_origin
+                else:
+                    new_dict[const.PROFILE_PARAM_VALUE_ORIGIN] = const.REPLACE_ME_PLACEHOLDER
                 # then insert the original, incoming values as values
                 if param_id in control_param_dict:
                     orig_param = control_param_dict[param_id]
@@ -172,9 +186,19 @@ def _construct_set_parameters_dict(
                 values = tmp_dict.get('values', None)
                 # if values are None then don´t display them in the markdown
                 if values is not None:
-                    new_dict = {'id': param_id, 'values': values, const.PROFILE_VALUES: ['<REPLACE_ME>']}
+                    new_dict = {
+                        'id': param_id,
+                        'values': values,
+                    }
                 else:
-                    new_dict = {'id': param_id, const.PROFILE_VALUES: ['<REPLACE_ME>']}
+                    new_dict = {
+                        'id': param_id,
+                    }
+                new_dict[const.PROFILE_VALUES] = [const.REPLACE_ME_PLACEHOLDER]
+                new_dict[const.PROFILE_PARAM_VALUE_ORIGIN] = const.REPLACE_ME_PLACEHOLDER
+            if param_value_origin is not None:
+                if context.purpose == ContextPurpose.PROFILE:
+                    new_dict[const.PARAM_VALUE_ORIGIN] = param_value_origin
             new_dict.pop('id', None)
             # validates if there are aggregated parameter values to the current parameter
             aggregated_props = [prop for prop in as_list(param_dict.props) if prop.name == const.AGGREGATES]
@@ -197,7 +221,9 @@ def _construct_set_parameters_dict(
                 const.AGGREGATES,
                 const.ALT_IDENTIFIER,
                 const.DISPLAY_NAME,
-                const.PROFILE_VALUES
+                const.PROFILE_VALUES,
+                const.PARAM_VALUE_ORIGIN,
+                const.PROFILE_PARAM_VALUE_ORIGIN
             )
             ordered_dict = {k: new_dict[k] for k in key_order if k in new_dict.keys()}
             set_param_dict[param_id] = ordered_dict