Merge pull request #222 from Dessia-tech/fix/debug

Fix/debug

.pylintrc

@@ -351,7 +351,7 @@ function-naming-style=snake_case
 #function-rgx=
 
 # Good variable names which should always be accepted, separated by a comma.
-good-names=i,j,k,n,ie,x,xi,x0,
+good-names=i,j,k,n,ie,x,xi,x0,v1,v2,
 	   ax,
            _
 

code_pylint.py

@@ -1,24 +1,27 @@
 '''
 Read pylint errors to see if number of errors does not exceed specified limits
-v1.1
+v1.2
 
 Changes:
     v1.1: move imports to top
+    v1.2: limit to 100 message to avoid overflow, global note check at end, ratchet effects
 '''
 
+
 import os
 import sys
+from pylint import __version__
 from pylint.lint import Run
 
-MIN_NOTE = 9.1
+MIN_NOTE = 9.17
 
 UNWATCHED_ERRORS = ['fixme', 'trailing-whitespace', 'import-error']
 
 MAX_ERROR_BY_TYPE = {
                      'protected-access': 26,
-                     'invalid-name': 18,
+                     'invalid-name': 8,
                      'consider-using-f-string': 10,
-                     'no-else-return': 17,
+                     'no-else-return': 4,
                      'arguments-differ': 12,
                      'no-member': 1,
                      'too-many-locals': 14,
@@ -28,7 +31,7 @@
                      'unused-argument': 6,
                      'cyclic-import': 11,
                      'no-self-use': 6,
-                     'unused-variable': 6,
+                     'unused-variable': 1,
                      'trailing-whitespace': 11,
                      'empty-docstring': 7,
                      'missing-module-docstring': 9,
@@ -84,6 +87,8 @@
                      'unsubscriptable-object': 0
                      }
 
+print('pylint version: ', __version__)
+
 f = open(os.devnull, 'w')
 
 old_stdout = sys.stdout
@@ -101,18 +106,13 @@
     pylint_note = results.linter.stats['global_note']
     PYLINT_OBJECT_STATS = False
 
-print('Pylint note: ', pylint_note)
-assert pylint_note >= MIN_NOTE
-print('You can increase MIN_NOTE in pylint to {} (actual: {})'.format(pylint_note,
-                                                                      MIN_NOTE))
-
 
 def extract_messages_by_type(type_):
     return [m for m in results.linter.reporter.messages if m.symbol == type_]
 
 
-# uncontrolled_errors = {}
 error_detected = False
+error_over_ratchet_limit = False
 
 if PYLINT_OBJECT_STATS:
     stats_by_msg = results.linter.stats.by_msg
@@ -126,21 +126,32 @@ def extract_messages_by_type(type_):
         else:
             max_errors = 0
 
+        # if number_errors < max_errors - RATCHET_ERRORS:
+        #     error_over_ratchet_limit = True
+
         if number_errors > max_errors:
             error_detected = True
             print('Fix some {} errors: {}/{}'.format(error_type,
                                                      number_errors,
                                                      max_errors))
-            for message in extract_messages_by_type(error_type):
+            for message in extract_messages_by_type(error_type)[:30]:
                 print('{} line {}: {}'.format(message.path, message.line, message.msg))
         elif number_errors < max_errors:
             print('You can lower number of {} to {} (actual {})'.format(
                 error_type, number_errors, max_errors))
 
-            # uncontrolled_errors[error_type] = number_errors
-
-# if uncontrolled_errors:
-#     print('Uncontrolled errors', uncontrolled_errors)
 
 if error_detected:
     raise RuntimeError('Too many errors\nRun pylint dessia_common to get the errors')
+
+if error_over_ratchet_limit:
+    raise RuntimeError('Please lower the error limits in code_pylint.py MAX_ERROR_BY_TYPE according to warnings above')
+
+print('Pylint note: ', pylint_note)
+# if pylint_note > MIN_NOTE + RATCHET_NOTE:
+#     raise ValueError(f'MIN_NOTE in code_pylint.py is too low, increase to at least {MIN_NOTE + RATCHET_NOTE}, max {pylint_note}')
+if pylint_note < MIN_NOTE:
+    raise ValueError(f'Pylint not is too low: {pylint_note}, expected {MIN_NOTE}')
+
+print('You can increase MIN_NOTE in pylint to {} (actual: {})'.format(pylint_note,
+                                                                      MIN_NOTE))

dessia_common/breakdown.py

@@ -31,39 +31,39 @@ def attrmethod_getter(object_, attr_methods):
     return object_
 
 
-def get_in_object_from_path(object_, path):
-    segments = path.lstrip('#/').split('/')
+def extract_from_object(object_, segment):
+    if is_sequence(object_):
+        return object_[int(segment)]
+
     if isinstance(object_, dict):
+        if segment in object_:
+            return object_[segment]
+
         try:
-            element = object_[segments[0]]
-        except KeyError:
-            msg = f'Cannot get in dict path {path}: end up @ {segments[0]}. Dict keys: {object_.keys()}'
-            raise RuntimeError(msg)
-    else:
-        element = getattr(object_, segments[0])
-
-    for segment in segments[1:]:
-        if is_sequence(element):
-            element = element[int(segment)]
-        elif isinstance(element, dict):  # A dict?
-            if segment in element:
-                element = element[segment]
-            else:
-                try:
-                    key = int(segment)
-                except ValueError:
-                    # should be a tuple
-                    key = []
-                    for subsegment in segment.strip('()').replace(' ', '').split(','):
-                        try:
-                            subkey = int(subsegment)
-                        except ValueError:
-                            subkey = subsegment
-                        key.append(subkey)
-                    key = tuple(key)
-                element = element[key]
-        else:
-            element = getattr(element, segment)
+            return object_[int(segment)]
+        except ValueError:
+            # should be a tuple
+            if segment.startswith('(') and segment.endswith(')') and ',' in segment:
+                key = []
+                for subsegment in segment.strip('()').replace(' ', '').split(','):
+                    try:
+                        subkey = int(subsegment)
+                    except ValueError:
+                        subkey = subsegment
+                    key.append(subkey)
+                return object_[tuple(key)]
+            # else:
+            raise NotImplementedError(f'Cannot extract segment {segment} from object {object_}')
+
+    # Finally, it is a regular object
+    return getattr(object_, segment)
+
+
+def get_in_object_from_path(object_, path):
+    segments = path.lstrip('#/').split('/')
+    element = object_
+    for segment in segments:
+        element = extract_from_object(element, segment)
 
     return element
 

dessia_common/graph.py

@@ -35,12 +35,11 @@ def cut_tree_final_branches(graph: nx.DiGraph):
 def explore_tree_from_leaves(graph: nx.DiGraph):
     exploration_order = []
     explored = {n: False for n in graph.nodes}
-    nn = graph.number_of_nodes()
-    # print('nn', nn)
+    number_nodes = graph.number_of_nodes()
     successors = {}
 
-    ns = 0
-    while nn:
+    # ns = 0
+    while number_nodes:
         found_node = False
         # Finding a starting node
         for node in graph.nodes:
@@ -51,7 +50,7 @@ def explore_tree_from_leaves(graph: nx.DiGraph):
                 else:
                     node_successors = list(graph.successors(node))
                     successors[node] = node_successors
-                    ns += 1
+                    # ns += 1
 
                 for out_node in node_successors:
                     if not explored[out_node]:
@@ -62,7 +61,7 @@ def explore_tree_from_leaves(graph: nx.DiGraph):
                     # Mark explored
                     explored[node] = True
                     exploration_order.append(node)
-                    nn -= 1
+                    number_nodes -= 1
                     found_node = True
                     break
         if not found_node:

dessia_common/utils/copy.py

@@ -13,23 +13,23 @@ def deepcopy_value(value, memo):
     if isinstance(value, type) or is_typing(value):  # For class
         return value
 
-    elif isinstance(value, (float, int, str)):
+    if isinstance(value, (float, int, str)):
         copied_value = value
         return copied_value
 
-    elif value is None:
+    if value is None:
         return None
 
-    elif value.__class__.__name__ in ['Point2D', 'Point3D',
-                                      'Vector2D', 'Vector3D']:
+    if value.__class__.__name__ in ['Point2D', 'Point3D',
+                                    'Vector2D', 'Vector3D']:
         try:
             copied_value = value.copy(deep=True, memo=memo)
         except TypeError:
             warnings.warn(f'{value.__class__.__name__}.copy() does not implement deep and memo arguments')
             copied_value = value.copy()
         return copied_value
 
-    elif isinstance(value, dc.DessiaObject):
+    if isinstance(value, dc.DessiaObject):
         memo_value = search_memo(value, memo)
         if memo_value is not None:
             return memo_value
@@ -42,10 +42,10 @@ def deepcopy_value(value, memo):
         memo[value] = copied_value
         return copied_value
 
-    elif isinstance(value, (dessia_common.files.BinaryFile, dessia_common.files.StringFile)):
+    if isinstance(value, (dessia_common.files.BinaryFile, dessia_common.files.StringFile)):
         return value.copy()
 
-    elif hasattr(value, '__deepcopy__'):
+    if hasattr(value, '__deepcopy__'):
         memo_value = search_memo(value, memo)
         if memo_value is not None:
             return memo_value
@@ -57,15 +57,13 @@ def deepcopy_value(value, memo):
         memo[value] = copied_value
         return copied_value
 
-    else:
-        if is_sequence(value):
-            return deepcopy_sequence(value, memo)
+    if is_sequence(value):
+        return deepcopy_sequence(value, memo)
 
-        elif isinstance(value, dict):
-            return deepcopy_dict(value, memo)
+    if isinstance(value, dict):
+        return deepcopy_dict(value, memo)
 
-        else:
-            raise NotImplementedError(f'unhandle type for copy: {value} of type {value.__class__}')
+    raise NotImplementedError(f'unhandle type for copy: {value} of type {value.__class__}')
 
 
 def deepcopy_dict(dict_value, memo):

dessia_common/utils/diff.py

@@ -31,7 +31,7 @@ def diff(value1, value2, path='#'):
         if isinstance(value1, float) and \
                 math.isclose(value1, value2, abs_tol=dc.FLOAT_TOLERANCE):
             return diff_values, missing_keys_in_other_object, invalid_types
-        elif value1 != value2:
+        if value1 != value2:
             diff_values.append((path, value1, value2))
         return diff_values, missing_keys_in_other_object, invalid_types
     elif isinstance(value1, dict):
@@ -69,8 +69,8 @@ def dict_diff(dict1, dict2, path='#'):
         if key not in dict2:
             missing_keys_in_other_object.append(key)
         else:
-            dk, mkk, itk = diff(value, dict2[key], path=path_key)
-            diff_values.extend(dk)
+            diff_key, mkk, itk = diff(value, dict2[key], path=path_key)
+            diff_values.extend(diff_key)
             missing_keys_in_other_object.extend(mkk)
             invalid_types.extend(itk)
 
@@ -85,8 +85,8 @@ def sequence_diff(seq1, seq2, path='#'):
     if len(seq1) != len(seq2):
         diff_values.append((path, seq1, seq2))
     else:
-        for iv, (v1, v2) in enumerate(zip(seq1, seq2)):
-            path_value = '{}/{}'.format(path, iv)
+        for i, (v1, v2) in enumerate(zip(seq1, seq2)):
+            path_value = '{}/{}'.format(path, i)
             dv, mkv, itv = diff(v1, v2, path=path_value)
             # print('dvs', dv, v1, v2)
             diff_values.extend(dv)
@@ -106,10 +106,8 @@ def data_eq(value1, value2):
     if isinstance_base_types(value1):
         if isinstance(value1, float):
             return math.isclose(value1, value2, abs_tol=dc.FLOAT_TOLERANCE)
-        else:
-            # if not(value1 == value2):
-            # print('base types', value1, value2, value1 == value2)
-            return value1 == value2
+
+        return value1 == value2
 
     if isinstance(value1, dict):
         return dict_data_eq(value1, value2)
@@ -141,24 +139,20 @@ def dict_data_eq(dict1, dict2):
 
     for key, value in dict1.items():
         if key not in dict2:
-            # print('missing key', key)
             return False
-        else:
-            if not data_eq(value, dict2[key]):
-                # print('dict key !=', key, value, dict2[key])
-                return False
+        if not data_eq(value, dict2[key]):
+            return False
     return True
 
 
 def sequence_data_eq(seq1, seq2):
     if len(seq1) != len(seq2):
-        # print('len diff', seq1, seq2)
         return False
-    else:
-        for v1, v2 in zip(seq1, seq2):
-            if not data_eq(v1, v2):
-                # print('seq false')
-                return False
+
+    for v1, v2 in zip(seq1, seq2):
+        if not data_eq(v1, v2):
+            # print('seq false')
+            return False
 
     return True