[NB] update array handling (#8869)

- update scalarizing
   - deal with model subscripts (just as any other subscript)
   - deal with scalar subscripts (include 1 as subscript for scalars instead of empty)

 - update entwining of equations
   - allow intersection to be empty

 - implement several dumping functions

Author: kabdelhak

OMCompiler/Compiler/NBackEnd/Classes/NBEquation.mo

@@ -274,10 +274,8 @@ public
             rest2 := intersectRest(iter2.name, start2, step2, stop2, start_max-step2, stop_min+step2);
         then (intersection, rest1, rest2);
 
-        else algorithm
-          Error.addMessage(Error.INTERNAL_ERROR,{getInstanceName() + " failed because only single iterators with equal step can be intersected:\n"
-            + Iterator.toString(iter1) + "\n" + Iterator.toString(iter2) + "\n"});
-        then fail();
+        // cannot intersect
+        else (EMPTY(), (iter1, EMPTY()), (EMPTY(), iter2));
       end match;
     end intersect;
 
@@ -1536,20 +1534,26 @@ public
 
     function entwine
       input list<Equation> eqn_lst        "has to be for-loops with combinable ranges";
+      input Integer nesting_level = 0;
       output list<Equation> entwined = {} "returns a single for-loop on top level if it is possible";
     protected
       Equation eqn1, eqn2, next;
       list<Equation> rest, tmp;
       Iterator intersection, rest1_left, rest1_right, rest2_left, rest2_right;
+      String shift = StringUtil.repeat("  ", nesting_level);
     algorithm
+      if Flags.isSet(Flags.DUMP_SLICE) then
+        print(shift + "[" + intString(nesting_level) + "] ### Entwining following equations:\n"
+          + List.toString(eqn_lst, function Equation.toString(str = shift + "  "), "", "", "\n", "\n\n"));
+      end if;
       eqn1 :: rest := eqn_lst;
       while not listEmpty(rest) loop
         eqn2 :: rest := rest;
         eqn1 := match (eqn1, eqn2)
 
           // entwine body if possible - equal iterator -> no intersecting
           case (FOR_EQUATION(), FOR_EQUATION()) guard(Iterator.isEqual(eqn1.iter, eqn2.iter)) algorithm
-            eqn1.body := entwine(listAppend(eqn1.body, eqn2.body));
+            eqn1.body := entwine(listAppend(eqn1.body, eqn2.body), nesting_level + 1);
           then eqn1;
 
           // if the iterators are not equal, they have to be intersected and the respective rests have to be handled
@@ -1566,7 +1570,7 @@ public
               tmp := FOR_EQUATION(
                 ty      = eqn1.ty,
                 iter    = intersection,
-                body    = entwine(listAppend(eqn1.body, eqn2.body)),
+                body    = entwine(listAppend(eqn1.body, eqn2.body), nesting_level + 1),
                 source  = eqn1.source,
                 attr    = eqn1.attr
               ) :: tmp;
@@ -1589,6 +1593,10 @@ public
         end match;
       end while;
       entwined := listReverse(eqn1 :: entwined);
+      if Flags.isSet(Flags.DUMP_SLICE) then
+        print(shift + "[" + intString(nesting_level) + "] +++ Result of entwining:\n"
+          + List.toString(entwined, function Equation.toString(str = shift  + "  "), "", "", "\n", "\n\n"));
+      end if;
     end entwine;
 
     function slice

OMCompiler/Compiler/NBackEnd/Classes/NBVariable.mo

@@ -1475,9 +1475,16 @@ public
       "Returns true if the variable is in the variable pointer array."
       input Pointer<Variable> var;
       input VariablePointers variables;
-      output Boolean b = getVarIndex(variables, getVarName(var)) > 0;
+      output Boolean b = containsCref(getVarName(var), variables);
     end contains;
 
+    function containsCref
+      "Returns true if a variable with this name is in the variable pointer array."
+      input ComponentRef cref;
+      input VariablePointers variables;
+      output Boolean b = getVarIndex(variables, cref) > 0;
+    end containsCref;
+
     function getVarNames
       "returns a list of crefs representing the names of all variables"
       input VariablePointers variables;
@@ -1578,7 +1585,7 @@ public
         // flatten potential arrays
         if Type.isArray(var.ty) then
           flattened := true;
-          scalar_vars := Scalarize.scalarizeVariable(var);
+          scalar_vars := Scalarize.scalarizeBackendVariable(var);
         else
           scalar_vars := {Pointer.access(var_ptr)};
         end if;

OMCompiler/Compiler/NBackEnd/Classes/NBackendDAE.mo

@@ -364,6 +364,7 @@ protected
     VariablePointers variables, unknowns, knowns, initials, auxiliaries, aliasVars, nonTrivialAlias;
     VariablePointers states, derivatives, algebraics, discretes, previous;
     VariablePointers parameters, constants;
+    Pointer<list<Pointer<Variable>>> binding_iter_lst = Pointer.create({});
     Boolean scalarized = Flags.isSet(Flags.NF_SCALARIZE);
   algorithm
     // instantiate variable data (with one more space for time variable);
@@ -458,8 +459,11 @@ protected
     parameters      := VariablePointers.fromList(parameters_lst, scalarized);
     constants       := VariablePointers.fromList(constants_lst, scalarized);
 
-    /* lower the variable bindings */
-    VariablePointers.map(variables, function lowerVariableBinding(variables = variables));
+    /* lower the variable bindings and add binding iterators */
+    variables       := VariablePointers.map(variables, function collectVariableBindingIterators(variables = variables, binding_iter_lst = binding_iter_lst));
+    variables       := VariablePointers.addList(Pointer.access(binding_iter_lst), variables);
+    knowns          := VariablePointers.addList(Pointer.access(binding_iter_lst), knowns);
+    variables       := VariablePointers.map(variables, function lowerVariableBinding(variables = variables));
 
     /* create variable data */
     variableData := BVariable.VAR_DATA_SIM(variables, unknowns, knowns, initials, auxiliaries, aliasVars, nonTrivialAlias,
@@ -549,17 +553,32 @@ protected
     end match;
   end lowerVariableKind;
 
+  function collectVariableBindingIterators
+    input output Variable var;
+    input VariablePointers variables;
+    input Pointer<list<Pointer<Variable>>> binding_iter_lst;
+  algorithm
+    _ := match var
+      local
+        Binding binding;
+      case Variable.VARIABLE(binding = binding as Binding.TYPED_BINDING()) algorithm
+        // collect all iterators (only locally known) so that they have a respective variable
+        Expression.map(binding.bindingExp, function collectBindingIterators(variables = variables, binding_iter_lst = binding_iter_lst));
+      then ();
+      else ();
+    end match;
+  end collectVariableBindingIterators;
+
   function lowerVariableBinding
     input output Variable var;
     input VariablePointers variables;
   algorithm
     var := match var
       local
         Binding binding;
-      case Variable.VARIABLE(binding = binding as Binding.TYPED_BINDING())
-        algorithm
-          binding.bindingExp := Expression.map(binding.bindingExp, function lowerComponentReferenceExp(variables = variables));
-          var.binding := binding;
+      case Variable.VARIABLE(binding = binding as Binding.TYPED_BINDING()) algorithm
+        binding.bindingExp := Expression.map(binding.bindingExp, function lowerComponentReferenceExp(variables = variables));
+        var.binding := binding;
       then var;
       else var;
     end match;
@@ -1106,6 +1125,23 @@ protected
     end try;
   end lowerComponentReference;
 
+  function collectBindingIterators
+    "collects all iterators in bindings and creates variables for them.
+    in bindings they are only known locally but they still need a respective variable"
+    input output Expression exp;
+    input VariablePointers variables;
+    input Pointer<list<Pointer<Variable>>> binding_iter_lst;
+  algorithm
+    _ := match exp
+      local
+        ComponentRef cref;
+      case Expression.CREF(cref = cref) guard(not VariablePointers.containsCref(cref, variables)) algorithm
+        Pointer.update(binding_iter_lst, lowerIterator(cref) :: Pointer.access(binding_iter_lst));
+      then ();
+      else ();
+    end match;
+  end collectBindingIterators;
+
 public
   function lowerComponentReferenceInstNode
     "Adds the pointer to a variable to a component reference. This function needs
@@ -1127,7 +1163,5 @@ public
     end match;
   end lowerComponentReferenceInstNode;
 
-protected
-
   annotation(__OpenModelica_Interface="backend");
 end NBackendDAE;

OMCompiler/Compiler/NBackEnd/Modules/1_Main/NBSorting.mo

@@ -60,6 +60,12 @@ public
       Integer mode          "solve mode index";
     end PSEUDO_BUCKET_KEY;
 
+    function toString
+      input PseudoBucketKey key;
+      output String str = "key: (start[s]: " + intString(key.eqn_start_idx)
+        + ", index[a]:" + intString(key.eqn_arr_idx) + ", mode: " + intString(key.mode) + ")";
+    end toString;
+
     function hash
       input PseudoBucketKey key   "the key to hash";
       input Integer modulo        "modulo value";
@@ -90,6 +96,26 @@ public
       array<list<Integer>> entwined_arr;
     end PSEUDO_BUCKET_ENTWINED;
 
+    function toString
+      input PseudoBucketValue val;
+      output String str;
+    algorithm
+      str := match val
+        local
+          PseudoBucketKey k;
+          PseudoBucketValue v;
+        case PSEUDO_BUCKET_SINGLE() then "\n\tval: (" + ComponentRef.toString(val.cref_to_solve)
+          + ", [" + intString(val.first_comp) + ":" + intString(val.last_comp) +"]" + ")";
+        case PSEUDO_BUCKET_ENTWINED() algorithm
+           str := "\n\tentwined:";
+           for tpl in val.entwined_lst loop
+            (k, v) := tpl;
+            str := str + "\n\t" + PseudoBucketKey.toString(k) + " :: " + PseudoBucketValue.toString(v);
+           end for;
+        then str;
+      end match;
+    end toString;
+
     function addIndices
       input output PseudoBucketValue val;
       input Integer eqn_scal_idx;
@@ -113,6 +139,11 @@ public
       array<Boolean> marks;
     end PSEUDO_BUCKET;
 
+    function toString
+      input PseudoBucket bucket;
+      output String str = UnorderedMap.toString(bucket.bucket, PseudoBucketKey.toString, PseudoBucketValue.toString);
+    end toString;
+
     function create
       "recollects subsets of multi-dimensional equations that have to be solved in the same way.
       currently only for loops!"
@@ -317,6 +348,11 @@ public
 
         // recollect array information
         bucket := PseudoBucket.create(comps_indices, matching.eqn_to_var, adj.mapping, adj.modes);
+
+        if Flags.isSet(Flags.DUMP_SLICE) then
+          print("--- BUCKETS:\n" + PseudoBucket.toString(bucket) + "\n\n");
+        end if;
+
         for idx_lst in comps_indices loop
           comp_opt := StrongComponent.createPseudo(idx_lst, matching.eqn_to_var, vars, eqns, adj.mapping, adj.modes, bucket);
           if Util.isSome(comp_opt) then

OMCompiler/Compiler/NBackEnd/Modules/2_Pre/NBRemoveSimpleEquations.mo

@@ -453,6 +453,11 @@ protected
           guard(BVariable.isParamOrConst(BVariable.getVarPointer(exp.cref)) or ComponentRef.isTime(exp.cref))
         then tpl;
 
+        // fail for multidimensional crefs for now
+        case Expression.CREF()
+          guard(BVariable.size(BVariable.getVarPointer(exp.cref)) > 1)
+        then FAILED_CREF_TPL;
+
         // variable found
         // 1. not time and not param or const
         // 2. less than two previous variables

OMCompiler/Compiler/NBackEnd/Util/NBSlice.mo

@@ -373,7 +373,7 @@ public
       var_arr_idx     := UnorderedMap.getSafe(stripped, map);
       (var_start, _)  := mapping.var_AtS[var_arr_idx];
       sizes           := ComponentRef.sizes(stripped);
-      subs            := list(Subscript.toExp(sub) for sub in ComponentRef.subscriptsAllWithWholeFlat(cref));
+      subs            := ComponentRef.subscriptsToExpression(cref);
       scal_lst        := combineFrames2Indices(var_start, sizes, subs, frames, UnorderedMap.new<Expression>(ComponentRef.hash, ComponentRef.isEqual));
 
       if listLength(scal_lst) <> eqn_size then
@@ -413,6 +413,7 @@ public
     list<ComponentRef> names;
     list<Expression> ranges, subs;
     list<list<Expression>> new_subs, new_row_cref_subs;
+    list<Subscript> evaluated_subs;
     list<tuple<ComponentRef, Expression>> frames;
     list<ComponentRef> new_row_crefs = {}, new_dep_crefs;
     list<list<ComponentRef>> scalar_dependenciesT = {};
@@ -422,19 +423,22 @@ public
     frames := List.zip(names, ranges);
 
     // get new subscripts for row cref
-    subs := list(Subscript.toExp(sub) for sub in ComponentRef.subscriptsAllWithWholeFlat(row_cref));
+    subs := ComponentRef.subscriptsToExpression(row_cref);
+    //subs := list(Subscript.toExp(sub) for sub in ComponentRef.subscriptsAllWithWholeFlat(row_cref));
     new_row_cref_subs := combineFrames2Exp(subs, frames, UnorderedMap.new<Expression>(ComponentRef.hash, ComponentRef.isEqual));
 
     // reapply new subscripts for each frame location
     stripped := ComponentRef.stripSubscriptsAll(row_cref);
     for new_subs_single in new_row_cref_subs loop
-      new_row_crefs := ComponentRef.mergeSubscripts(list(Subscript.fromTypedExp(exp) for exp in new_subs_single), stripped) :: new_row_crefs;
+      evaluated_subs := list(Subscript.fromTypedExp(exp) for exp in new_subs_single);
+      new_row_crefs := ComponentRef.mergeSubscripts(evaluated_subs, stripped, false, true) :: new_row_crefs;
     end for;
 
     // get the scalar crefs for each cref
     if not listEmpty(dependencies) then
       for cref in dependencies loop
-        subs     := list(Subscript.toExp(sub) for sub in ComponentRef.subscriptsAllWithWholeFlat(cref));
+        subs := ComponentRef.subscriptsToExpression(cref);
+        //subs     := list(Subscript.toExp(sub) for sub in ComponentRef.subscriptsAllWithWholeFlat(cref));
         new_subs := combineFrames2Exp(subs, frames, UnorderedMap.new<Expression>(ComponentRef.hash, ComponentRef.isEqual));
 
         if listLength(new_subs) <> listLength(new_row_crefs) then
@@ -448,7 +452,8 @@ public
         new_dep_crefs := {};
         stripped := ComponentRef.stripSubscriptsAll(cref);
         for new_subs_single in new_subs loop
-          new_dep_crefs := ComponentRef.mergeSubscripts(list(Subscript.fromTypedExp(exp) for exp in new_subs_single), stripped) :: new_dep_crefs;
+          evaluated_subs := list(Subscript.fromTypedExp(exp) for exp in new_subs_single);
+          new_dep_crefs := ComponentRef.mergeSubscripts(evaluated_subs, stripped, false, true) :: new_dep_crefs;
         end for;
         scalar_dependenciesT := new_dep_crefs :: scalar_dependenciesT;
       end for;
@@ -848,6 +853,7 @@ protected
         ComponentRef iterator;
         Expression range;
         Integer start, step, stop;
+        list<Expression> local_subs;
 
       // only occurs for non-for-loop equations (no frames to replace)
       case {} then {subs};
@@ -860,7 +866,8 @@ protected
           UnorderedMap.add(iterator, Expression.INTEGER(index), replacements);
           if listEmpty(rest) then
             // bottom line, resolve current configuration and create index for it
-            new_subs := list(SimplifyExp.simplify(Expression.map(sub, function Replacements.applySimpleExp(replacements = replacements))) for sub in subs) :: new_subs;
+            local_subs := list(SimplifyExp.simplify(Expression.map(sub, function Replacements.applySimpleExp(replacements = replacements))) for sub in subs);
+            new_subs := listReverse(local_subs) :: new_subs;
           else
             // not last frame, go deeper
             new_subs := combineFrames2Exp(subs, rest, replacements, new_subs);