Address feedback and fix tests

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

src/licensedcode/models.py

@@ -24,8 +24,11 @@
 from licensedcode._vendor import attr
 from license_expression import ExpressionError
 from license_expression import Licensing
-from saneyaml import load as yaml_load
-from saneyaml import dump as yaml_dump
+from saneyaml import load as saneyaml_load
+from saneyaml import dump as saneyaml_dump
+from yaml import load as yaml_load
+from yaml import dump as yaml_dump
+from yaml import CSafeLoader
 
 from commoncode.fileutils import file_base_name
 from commoncode.fileutils import file_name
@@ -664,9 +667,17 @@ def validate(licenses, verbose=False, no_dupe_urls=False):
 
             data = {"text": text}
             # We are testing whether we can dump as yaml and load from yaml
-            # without failing (i.e. whether the text is yaml safe) 
-            yaml_string = yaml_dump(data, indent=4)
-            loaded_yaml = yaml_load(yaml_string)
+            # without failing (i.e. whether the text is yaml safe)
+            # Using saneyaml
+            try:
+                yaml_string = saneyaml_dump(data, indent=4)
+                loaded_yaml = saneyaml_load(yaml_string)
+            except Exception:
+                errors['GLOBAL'].append(
+                    f'Error invalid YAML text at: {lic.key}, failed during saneyaml.load()'
+                )
+            # This fails because of missing line break at text end, added by saneyaml_dump
+            # assert text == loaded_yaml["text"]
 
             license_itokens = tuple(index_tokenizer(text))
             if not license_itokens:
@@ -750,9 +761,9 @@ def validate(licenses, verbose=False, no_dupe_urls=False):
 def get_yaml_safe_text(text):
 
     data = {"text": text}
-    yaml_string = yaml_dump(data, indent=4)
+    yaml_string = saneyaml_dump(data, indent=4)
     try:
-        loaded_yaml = yaml_load(yaml_string)
+        loaded_yaml = saneyaml_load(yaml_string)
     except Exception:
         text = text.replace('\n\n', '\n \n')
     return text
@@ -1811,6 +1822,16 @@ def validate(self, licensing=None):
                 if len(set(self.referenced_filenames)) != len(self.referenced_filenames):
                     yield 'referenced_filenames cannot contain duplicates.'
 
+        text = self.text
+        data = {"text": text}
+        # We are testing whether we can dump as yaml and load from yaml
+        # without failing (i.e. whether the text is yaml safe)
+        try:
+            yaml_string = saneyaml_dump(data, indent=4)
+            loaded_yaml = saneyaml_load(yaml_string)
+        except Exception:
+            yield (f'Error invalid YAML text at: {self.identifier}, failed during saneyaml.load()')
+
     def license_keys(self, unique=True):
         """
         Return a list of license keys for this rule.

tests/formattedcode/data/yaml/package-and-licenses-expected.yaml

@@ -33,6 +33,10 @@ headers:
         platform_version: '#67-Ubuntu SMP Mon Mar 13 14:22:10 UTC 2023'
         python_version: "3.8.10 (default, Mar 13 2023, 10:26:41) \n[GCC 9.4.0]"
       spdx_license_list_version: '3.20'
+      additional_license_directory: /home/ayansinha/nexB/write_access/scancode-extra/tests/licensedcode/data/additional_licenses/additional_dir
+      additional_license_plugins:
+        - /home/ayansinha/nexB/write_access/scancode-extra/venv/lib/python3.8/site-packages/licenses_to_install1
+        - /home/ayansinha/nexB/write_access/scancode-extra/venv/lib/python3.8/site-packages/licenses_to_install2
       files_count: 4
 summary:
   declared_license_expression: apache-2.0 AND (apache-2.0 OR mit)
@@ -85,6 +89,7 @@ packages:
     code_view_url:
     vcs_url:
     copyright:
+    holder:
     declared_license_expression: apache-2.0
     declared_license_expression_spdx: Apache-2.0
     license_detections:
@@ -1137,6 +1142,7 @@ files:
         code_view_url:
         vcs_url:
         copyright:
+        holder:
         declared_license_expression: apache-2.0
         declared_license_expression_spdx: Apache-2.0
         license_detections:

tests/licensedcode/test_models.py

@@ -185,6 +185,9 @@ def test_rule_from_license_have_text_file_and_data_file_are_computed_correctly(s
 class TestRule(FileBasedTesting):
     test_data_dir = TEST_DATA_DIR
 
+    def test_validate_license_rules_data(self):
+        rules = list(models.get_rules(validate=True))
+
     def test_create_rule_ignore_punctuation(self):
         test_rule = create_rule_from_text_and_expression(text='A one. A two. A three.')
         expected = ['one', 'two', 'three']