Merge branch 'develop' into docs/build_gen_oscal

scripts/fix_any.py

@@ -26,49 +26,17 @@ class to the singular form using a manually curated lookup table.
 import logging
 import operator
 import re
-import string
 
 from trestle.oscal import OSCAL_VERSION_REGEX
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 logger.addHandler(logging.StreamHandler())
 
-# these plurals need to be fixed in Dict even when non-optional
-special_plurals = ['Users', 'Components']
-
-plural_lut = {
-    'Annotations': 'Annotation',
-    'AssessmentSubjects': 'AssessmentSubject',
-    'ByComponents': 'ByComponent',
-    'Capabilities': 'Capability',
-    'Components': 'SystemComponent',
-    'Controls': 'Control',
-    'Diagrams': 'Diagram',
-    'EmailAddresses': 'EmailStr',
-    'ImplementedComponents': 'ImplementedComponent',
-    'IncorporatesComponents': 'IncorporatesComponent',
-    'IncorporatesComponentDefnitions': 'IncorporatesComponentDefinition',
-    'IncorporatesTargets': 'IncorporatesTarget',
-    'InformationTypeIds': 'InformationTypeId',
-    'InventoryItems': 'InventoryItem',
-    'Objectives': 'Objective',
-    'ParameterSettings': 'SetParameter',
-    'ResponsibleParties': 'ResponsibleParty',
-    'ResponsibleRoles': 'ResponsibleRole',
-    'SetParameters': 'SetParameter',
-    'Statements': 'Statement',
-    'Tools': 'Tool',
-    'Targets': 'DefinedTarget',
-    'Users': 'SystemUser'
-}
-
-# Class names with this substring are due to confusion in oscal that will go away.
-# For now delete the classes and convert the corresponding unions to conlist
-singleton_suffixes = ['GroupItem', 'Item']
-singleton_prefixes = ['Include', 'Exclude']
-numbered_classes = ['Base64']
+bad_items = ['RelevantEvidence', 'Provided', 'Inherited', 'Satisfied']
+base64_str = 'Base64'
 class_header = 'class '
+
 license_header = (
     '# -*- mode:python; coding:utf-8 -*-\n'
     '# Copyright (c) 2020 IBM Corp. All rights reserved.\n'
@@ -114,22 +82,9 @@ def add_line(self, line):
         self.lines.append(line)
 
     def add_ref_if_good(self, ref_name):
-        """Add refs after removing digits and/or singleton string."""
-        if not ref_name:
-            return
-        if ref_name == 'Base64':
+        """Add non-empty refs."""
+        if ref_name:
             self.refs.add(ref_name)
-            return
-        for suffix in singleton_suffixes:
-            n = ref_name.find(suffix)
-            if n > 0:
-                ref_name = ref_name[:n]
-                break
-        for prefix in singleton_prefixes:
-            n = ref_name.find(prefix)
-            if n == 0 and ref_name != prefix:
-                ref_name = ref_name[len(prefix):]
-        self.refs.add(ref_name.rstrip(string.digits))
 
     def add_ref_pattern(self, p, line):
         """Add new class names found based on pattern."""
@@ -216,25 +171,10 @@ def find_order(self, class_text_list):
 
     def is_good(self):
         """Is this class on that should be retained in output."""
-        for line in self.lines:
-            """
-            # for now leave __root__ files since some are still needed
-            if line.find('__root__: str') >= 0:
-               return False
-            # But remove all pass files
-            """
-            if line.find('pass') >= 0:
-                return False
         if not self.name:
             return False
         if self.name.find('min_items') >= 0:
             return False
-        # classes like Base64 are ok
-        if self.name in numbered_classes:
-            return True
-        # but any class ending with digit is assumed bad
-        if self.name != self.name.rstrip(string.digits):
-            return False
         return True
 
 
@@ -301,17 +241,13 @@ def reorder(class_list):
     return class_list, forward_refs
 
 
-def fix_header(header, needs_conlist):
+def fix_header(header):
     """Fix imports and remove timestamp from header."""
+    '--disable-timestamp should work but had no effect.'
     new_header = []
     for r in header:
-        # Any is needed for biblio so still need to import it
-        # r = re.sub(' Any, ', ' ', r) # noqa: E800
         if r.find(' timestamp: ') >= 0:
             continue
-        if needs_conlist:
-            # add import of conlist for Union[A, A] case
-            r = re.sub(r'^(from\s+pydantic\s+import.*)$', r'\1, conlist', r)
         new_header.append(r)
     return new_header
 
@@ -329,19 +265,14 @@ def clean_classes(class_list):
     """Clean all lines of text in each class for endings that should be removed."""
     for j in range(len(class_list)):
         cls = class_list[j]
-        for singleton in singleton_suffixes + singleton_prefixes:
-            if cls.name != singleton:
-                cls.name = cls.name.replace(singleton, '')
-            for i in range(len(cls.lines)):
-                line = cls.lines[i]
-                if line.find('class ') != 0:
-                    if line.find(f'[{singleton}]') == -1:
-                        line = line.replace(singleton, '')
-                for c2 in class_list:
-                    cname = c2.name
-                    pattern = cname + '[0-9]*'
-                    line = re.sub(pattern, cname, line)
-                cls.lines[i] = line
+        for i in range(len(cls.lines)):
+            line = cls.lines[i]
+            for bad_item in bad_items:
+                line = line.replace(f'{bad_item}Item', bad_item)
+            cls.lines[i] = line
+        for bad_item in bad_items:
+            if cls.name == f'{bad_item}Item':
+                cls.name = bad_item
         class_list[j] = cls
     return class_list
 
@@ -355,7 +286,7 @@ def fix_bad_minitems(class_list):
             neq = line.find('= Field(None')
             if neq > 0 and line.find(cls.name) >= 0:
                 cls.lines[i] = line[:neq] + '= None'
-            if neq > 0 and line.find('List[Base64]') >= 0:
+            if neq > 0 and line.find(f'List[{base64_str}]') >= 0:
                 cls.lines[i] = line[:neq] + '= None'
         class_list[j] = cls
     return class_list
@@ -384,7 +315,6 @@ def fix_file(fname):
 
     class_text = None
     done_header = False
-    needs_conlist = False
 
     # Find Any's and replace with appropriate singular class
     # Otherwise accumulate all class dependencies for reordering with no forward refs
@@ -395,38 +325,16 @@ def fix_file(fname):
             elif r.find(class_header) == 0:  # start of new class
                 done_header = True
                 if class_text is not None:  # we are done with current class so add it
-                    # prevent anomalous singletons from appearing in output
-                    for singleton in singleton_prefixes + singleton_suffixes:
-                        if class_text.name.find(singleton) >= 0 and class_text.name != singleton:
-                            class_text.name = class_text.name.replace(singleton, '')
-                            for i in range(len(class_text.lines)):
-                                class_text.lines[i] = class_text.lines[i].replace(singleton, '')
                     all_classes.append(class_text)
                 class_text = ClassText(r)
             else:
                 if not done_header:  # still in header
                     header.append(r.rstrip())
                 else:
-                    # fix any line containing Union[A, A] to Union[A, conlist(A, min_items=2)]
-                    # this may now include Unions that previously involved anomalous singletons
-                    r = re.sub(r'Union\[([^,]*),\s*\1\]', r'Union[\1, conlist(\1, min_items=2)]', r)
-                    if r.find(', conlist(') >= 0:
-                        needs_conlist = True
-                    if r.find('Optional[Dict[str, ') >= 0:
-                        match = re.search(r'Optional\[Dict\[str, (.+?)\]\]', r)
-                        if match:
-                            plural = match.group(1)
-                            if plural != 'Any' and plural in plural_lut:
-                                r = r.replace(plural, plural_lut[plural])
-                    for special in special_plurals:
-                        if r.find(f'Dict[str, {special}]') >= 0:
-                            r = r.replace(special, plural_lut[special])
                     p = re.compile(r'.*Optional\[Union\[([^,]+),.*List\[Any\]')
                     refs = p.findall(r)
                     if len(refs) == 1:
                         r = r.replace('List[Any]', f'List[{refs[0]}]')
-                    # mark regex strings as raw
-                    r = re.sub(r"(\s*regex\s*=\s*)\'(.*)", r"\1r'\2", r)
                     class_text.add_all_refs(r)
                     class_text.add_line(r.rstrip())
 
@@ -447,7 +355,7 @@ def fix_file(fname):
     # then find any required forward refs and replace the original list with the new ones
     reordered_classes, forward_refs = reorder(sorted_classes)
 
-    header = fix_header(header, needs_conlist)
+    header = fix_header(header)
 
     # write the classes out in the fixed order
     with open(fname, 'w') as out_file:

scripts/order_classes.py

@@ -0,0 +1,65 @@
+# -*- mode:python; coding:utf-8 -*-
+
+# Copyright (c) 2020 IBM Corp. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Script to allow comparison of files before and after applying fix_any.py.
+
+Load every class in a file as lines of text and output them sorted by class name.
+This allows a standard diff to reveal changes in class content without regard to order.
+Missing classes are also revealed.
+"""
+import operator
+import sys
+
+from fix_any import ClassText
+
+all_classes = []
+header = []
+forward_refs = []
+class_header = 'class '
+class_text = None
+done_header = False
+
+if len(sys.argv) < 2:
+    exit()
+
+fname = sys.argv[1]
+with open(fname, 'r') as infile:
+    for r in infile.readlines():
+        if r.find('.update_forward_refs()') >= 0:
+            forward_refs.append(r)
+        elif r.find(class_header) == 0:  # start of new class
+            done_header = True
+            if class_text is not None:  # we are done with current class so add it
+                all_classes.append(class_text)
+            class_text = ClassText(r)
+        else:
+            if not done_header:  # still in header
+                header.append(r.rstrip())
+            else:
+                class_text.add_line(r.rstrip())
+
+all_classes.append(class_text)  # don't forget final class
+
+sorted_classes = sorted(all_classes, key=operator.attrgetter('name'))
+
+sorted_forwards = sorted(forward_refs)
+
+outname = fname.replace('.py', '_sorted.py')
+
+with open(outname, 'w') as f:
+    for c in sorted_classes:
+        f.writelines('\n'.join(c.lines) + '\n')
+    f.writelines(sorted_forwards)

setup.cfg

@@ -28,7 +28,7 @@ install_requires =
     paramiko
     pyyaml
     furl
-    pydantic[email]==1.7.* # Temporary fix see issue #381
+    pydantic[email]>=1.8.1
     python-dotenv>=0.10.4
     datamodel-code-generator[http]