Preserve parentheses during parsing (#8125)

* Parse comments into the AST: Tuples, expressions, elements and classes now store comments
* Disabled some diff algorithm parts that are no longer necessary
* Parse comments before and after the end of the class (after annotations) into the AST
* Add special case for diffs and whitespace after newline

.openmodelica.aspell

@@ -1,4 +1,4 @@
-personal_ws-1.1 en 79 UTF-8
+personal_ws-1.1 en 80 UTF-8
 subscripted
 UTF
 RML
@@ -77,3 +77,4 @@ SHA
 differentiable
 evaluable
 convertPackageToLibrary
+noneFromVersion

OMCompiler/Compiler/FrontEnd/Absyn.mo

@@ -121,6 +121,8 @@ uniontype Class
     Boolean     encapsulatedPrefix "true if encapsulated" ;
     Restriction restriction  "Restriction" ;
     ClassDef    body;
+    list<String> commentsBeforeEnd;
+    list<String> commentsAfterEnd;
     Info       info    "Information: FileName is the class is defined in +
                isReadOnly bool + start line no + start column no +
                end line no + end column no";
@@ -605,6 +607,10 @@ uniontype ElementArg "Wrapper for things that modify elements, modifications and
     Info info "needed because ElementSpec does not contain this info; Element does";
   end REDECLARATION;
 
+  record ELEMENTARGCOMMENT "A lexer comment"
+    String comment;
+  end ELEMENTARGCOMMENT;
+
 end ElementArg;
 
 public
@@ -810,6 +816,12 @@ uniontype Exp "The Exp uniontype is the container of a Modelica expression.
     Exp index;
   end DOT;
 
+  record EXPRESSIONCOMMENT
+    list<String> commentsBefore;
+    Exp exp;
+    list<String> commentsAfter;
+  end EXPRESSIONCOMMENT;
+
 end Exp;
 
 uniontype Case "case in match or matchcontinue"

OMCompiler/Compiler/FrontEnd/AbsynToSCode.mo

@@ -337,24 +337,9 @@ protected function containsExternalFuncDecl
 algorithm
   outBoolean := match (inClass)
     local
-      Boolean res,b,c,d;
-      String a;
-      Absyn.Restriction e;
-      list<Absyn.ClassPart> rest;
-      Option<String> cmt;
-      SourceInfo file_info;
-      list<Absyn.Annotation> ann;
-    case (Absyn.CLASS(body = Absyn.PARTS(classParts = (Absyn.EXTERNAL() :: _)))) then true;
-    case (Absyn.CLASS(name = a,partialPrefix = b,finalPrefix = c,encapsulatedPrefix = d,restriction = e,
-                      body = Absyn.PARTS(classParts = (_ :: rest),comment = cmt,ann=ann),info = file_info))
-      then containsExternalFuncDecl(Absyn.CLASS(a,b,c,d,e,Absyn.PARTS({},{},rest,ann,cmt),file_info));
-    /* adrpo: handling also the case model extends X external ... end X; */
-    case (Absyn.CLASS(body = Absyn.CLASS_EXTENDS(parts = (Absyn.EXTERNAL() :: _)))) then true;
-    /* adrpo: handling also the case model extends X external ... end X; */
-    case (Absyn.CLASS(name = a,partialPrefix = b,finalPrefix = c,encapsulatedPrefix = d,restriction = e,
-                      body = Absyn.CLASS_EXTENDS(parts = (_ :: rest),comment = cmt,ann=ann),
-                      info = file_info))
-      then containsExternalFuncDecl(Absyn.CLASS(a,b,c,d,e,Absyn.PARTS({},{},rest,ann,cmt),file_info));
+      list<Absyn.ClassPart> parts;
+    case (Absyn.CLASS(body = Absyn.PARTS(classParts = parts))) then List.exist(parts,AbsynUtil.isExternalPart);
+    case (Absyn.CLASS(body = Absyn.CLASS_EXTENDS(parts = parts))) then List.exist(parts,AbsynUtil.isExternalPart);
     else false;
   end match;
 end containsExternalFuncDecl;
@@ -1776,6 +1761,7 @@ algorithm
               elem));
         then
           sub :: subMods;
+      case Absyn.ELEMENTARGCOMMENT() then subMods;
     end match;
   end for;
 

OMCompiler/Compiler/FrontEnd/AbsynUtil.mo

@@ -323,6 +323,12 @@ algorithm
       then
         (if referenceEq(exp.exp,e1) and referenceEq(exp.index,e2) then exp else Absyn.DOT(e1, e2), arg);
 
+    case Absyn.EXPRESSIONCOMMENT()
+      equation
+        (e1, arg) = traverseExpBidir(exp.exp, enterFunc, exitFunc, arg);
+      then
+        (if referenceEq(exp.exp,e1) then exp else Absyn.EXPRESSIONCOMMENT(exp.commentsBefore, e1, exp.commentsAfter), arg);
+
     else
       algorithm
         (,,enterName) := System.dladdr(enterFunc);
@@ -662,6 +668,8 @@ algorithm
       then
         ();
 
+    case Absyn.EQUATIONITEMCOMMENT() then ();
+
   end match;
 end traverseEquationItemBidir;
 
@@ -1908,6 +1916,9 @@ algorithm
       // inExp.index is only allowed to contain names to index the function call; not crefs that are evaluated in any way
       then getCrefFromExp(inExp.exp,includeSubs,includeFunctions);
 
+    case (Absyn.EXPRESSIONCOMMENT(),_,_)
+      then getCrefFromExp(inExp.exp,includeSubs,includeFunctions);
+
     else
       equation
         Error.addInternalError(getInstanceName() + " failed " + Dump.printExpStr(inExp), sourceInfo());
@@ -2408,6 +2419,33 @@ algorithm
   end match;
 end getSubsFromCref;
 
+public function getString
+  input Absyn.Exp exp;
+  output String str;
+algorithm
+  str := match exp
+    case Absyn.EXPRESSIONCOMMENT() then getString(exp.exp);
+    case Absyn.STRING(str) then str;
+  end match;
+end getString;
+
+public function stripCommentExpressions
+  input output Absyn.Exp exp;
+algorithm
+  (exp,_) := traverseExp(exp, stripCommentExpressionsHelper, true);
+end stripCommentExpressions;
+
+protected function stripCommentExpressionsHelper<T>
+  input output Absyn.Exp exp;
+  input output T extra;
+algorithm
+  exp := match exp
+    case Absyn.TUPLE({exp}) then exp;
+    case Absyn.EXPRESSIONCOMMENT() then exp.exp;
+    else exp;
+  end match;
+end stripCommentExpressionsHelper;
+
 public function crefGetLastIdent
   "Gets the last ident in a Absyn.ComponentRef"
   input Absyn.ComponentRef cref;
@@ -3694,7 +3732,7 @@ algorithm
   outExternal := List.find(class_parts, isExternalPart);
 end getExternalDecl;
 
-protected function isExternalPart
+public function isExternalPart
   input Absyn.ClassPart inClassPart;
   output Boolean outFound;
 algorithm
@@ -5608,6 +5646,13 @@ algorithm
   callExp := Absyn.Exp.CALL(name, Absyn.FunctionArgs.FUNCTIONARGS(posArgs, namedArgs), {});
 end makeCall;
 
+public function setClassCommentsAfterEnd
+  input output Absyn.Class cl;
+  input list<String> comments;
+algorithm
+  cl.commentsAfterEnd := comments;
+end setClassCommentsAfterEnd;
+
 public function pathReplaceFirst
   "Replaces the first identifier of a path with another path. Ex:
     pathReplaceFirst(A.B.C, X.Y.Z) => X.Y.Z.B.C"

OMCompiler/Compiler/FrontEnd/ClassLoader.mo

@@ -320,12 +320,13 @@ algorithm
         opt_cl = parsePackageFile(packagefile, strategy, true, within_, id, encrypted);
         // print("Got " + packagefile + "\n");
         if (isSome(opt_cl)) then
-          (class_ as Absyn.CLASS(name,pp,fp,ep,r,Absyn.PARTS(tv,ca,cp,ann,cmt),info)) = Util.getOption(opt_cl);
+          (class_ as Absyn.CLASS(body=Absyn.PARTS(tv,ca,cp,ann,cmt))) = Util.getOption(opt_cl);
           reverseOrder = getPackageContentNames(class_, orderfile, mp_1, Error.getNumErrorMessages(), encrypted);
           path = AbsynUtil.joinWithinPath(within_,Absyn.IDENT(id));
           w2 = Absyn.WITHIN(path);
           cp = List.fold4(reverseOrder, loadCompletePackageFromMp2, mp_1, strategy, w2, encrypted, {});
-          opt_cl = SOME(Absyn.CLASS(name,pp,fp,ep,r,Absyn.PARTS(tv,ca,cp,ann,cmt),info));
+          class_.body = Absyn.PARTS(tv,ca,cp,ann,cmt);
+          opt_cl = SOME(class_);
         end if;
       then opt_cl;
     case (_,pack,mp,_)

OMCompiler/Compiler/FrontEnd/Dump.mo

@@ -173,7 +173,7 @@ algorithm
   // -m, --modelicaOutput       Enables valid modelica output for flat modelica.
   status := Flags.getConfigBool(Flags.MODELICA_OUTPUT);
   FlagsUtil.setConfigBool(Flags.MODELICA_OUTPUT, false);
-  outString := Tpl.tplString2(AbsynDumpTpl.dumpClass, inClass, defaultDumpOptions);
+  outString := Tpl.tplString3(AbsynDumpTpl.dumpClass, inClass, "", defaultDumpOptions);
   FlagsUtil.setConfigBool(Flags.MODELICA_OUTPUT, status);
 end unparseClassStr;
 
@@ -816,6 +816,34 @@ algorithm
   FlagsUtil.setConfigBool(Flags.MODELICA_OUTPUT, status);
 end unparseElementArgStr;
 
+public function shouldSeparateAfterElementArg
+  input list<Absyn.ElementArg> args;
+  output list<tuple<Absyn.ElementArg,Boolean>> outArgs;
+protected
+  Integer numNonComment=0, cur=0;
+  Boolean b;
+algorithm
+  for arg in args loop
+    numNonComment := match arg
+      case Absyn.ELEMENTARGCOMMENT() then numNonComment;
+      else numNonComment + 1;
+    end match;
+  end for;
+  outArgs := {};
+  for arg in args loop
+    b := match arg
+      case Absyn.ELEMENTARGCOMMENT() then false;
+      else
+        algorithm
+          cur := cur + 1;
+        then cur < numNonComment;
+    end match;
+    outArgs := (arg,b)::outArgs;
+  end for;
+  outArgs := listReverse(outArgs);
+end shouldSeparateAfterElementArg;
+
+
 public function unparseElementItemStr
   "Prettyprints and ElementItem."
   input Absyn.ElementItem inElementItem;
@@ -3353,7 +3381,7 @@ algorithm
       Absyn.Restriction restriction;
       Absyn.ClassDef    body;
       SourceInfo info;
-    case Absyn.CLASS(name,partialPrefix,finalPrefix,encapsulatedPrefix,restriction,body,info)
+    case Absyn.CLASS(name,partialPrefix,finalPrefix,encapsulatedPrefix,restriction,body,_,_,info)
       equation
         Print.printBuf("record Absyn.CLASS name = \"");
         Print.printBuf(name);

OMCompiler/Compiler/FrontEnd/InstSection.mo

@@ -949,11 +949,13 @@ protected function checkTupleCallEquationMessage
   input Absyn.Exp right;
   input SourceInfo info;
 algorithm
-  _ := match (left, right)
+  _ := match (AbsynUtil.stripCommentExpressions(left), AbsynUtil.stripCommentExpressions(right))
     local
       list<Absyn.Exp> crefs;
       String left_str, right_str;
 
+    case (Absyn.TUPLE({_}), _) then ();
+
     case (Absyn.TUPLE(crefs), Absyn.CALL())
       algorithm
         if not List.all(crefs, AbsynUtil.isCref) then
@@ -4971,8 +4973,22 @@ protected function instAssignment2
   input Integer numError;
   output FCore.Cache outCache;
   output list<DAE.Statement> stmts "more statements due to loop unrolling";
+protected
+  Absyn.Exp varNoComment, inRhsNoComment;
 algorithm
-  (outCache,stmts) := matchcontinue (inCache,var,value,props)
+  varNoComment := AbsynUtil.stripCommentExpressions(var);
+  inRhsNoComment := AbsynUtil.stripCommentExpressions(inRhs);
+  _ := match varNoComment
+    local
+      Absyn.Exp lhs;
+    case Absyn.TUPLE({lhs})
+      algorithm
+        (outCache,stmts) := instAssignment2(inCache,inEnv,inIH,inPre,lhs,inRhsNoComment,value,props,info,inSource,initial_,inImpl,unrollForLoops,numError);
+        return;
+      then ();
+    else ();
+  end match;
+  (outCache,stmts) := matchcontinue (inCache,varNoComment,value,props)
     local
       DAE.ComponentRef ce,ce_1;
       DAE.Properties cprop,eprop,prop,prop1,prop2;
@@ -5000,7 +5016,7 @@ algorithm
           Static.elabCrefNoEval(cache, inEnv, cr, inImpl, false, inPre, info);
         DAE.T_ARRAY( dims = {_}) = t;
         rhs = e_1;
-        Static.checkAssignmentToInput(var, attr, inEnv, false, info);
+        Static.checkAssignmentToInput(varNoComment, attr, inEnv, false, info);
         DAE.T_ARRAY(dims = lhs_dim :: _) = Expression.typeof(lhs);
         DAE.T_ARRAY(dims = rhs_dim :: _) = Expression.typeof(rhs);
         {} = expandArrayDimension(lhs_dim, lhs);
@@ -5013,7 +5029,7 @@ algorithm
       equation
         (cache,DAE.CREF(ce,t),cprop,attr) =
           Static.elabCrefNoEval(cache, inEnv, cr, inImpl, false, inPre, info);
-        Static.checkAssignmentToInput(var, attr, inEnv, false, info);
+        Static.checkAssignmentToInput(varNoComment, attr, inEnv, false, info);
         (cache, ce_1) = Static.canonCref(cache, inEnv, ce, inImpl);
         (cache, ce_1) = PrefixUtil.prefixCrefInnerOuter(cache, inEnv, inIH, ce_1, inPre);
 
@@ -5055,7 +5071,7 @@ algorithm
       equation
         (cache,cre,cprop,attr) =
           Static.elabCrefNoEval(cache,inEnv, cr, inImpl,false,inPre,info);
-        Static.checkAssignmentToInput(var, attr, inEnv, false, info);
+        Static.checkAssignmentToInput(varNoComment, attr, inEnv, false, info);
         (cache,cre2) = PrefixUtil.prefixExp(cache, inEnv, inIH, cre, inPre);
         (cache, e_1, eprop) = Ceval.cevalIfConstant(cache, inEnv, e_1, eprop, inImpl, info);
         (cache,e_2) = PrefixUtil.prefixExp(cache, inEnv, inIH, e_1, inPre);
@@ -5134,7 +5150,7 @@ algorithm
 
     case (cache,e1 as Absyn.TUPLE(expressions = expl),_,prop2)
       equation
-        Absyn.CALL() = inRhs;
+        Absyn.CALL() = inRhsNoComment;
         true = List.all(expl, AbsynUtil.isCref);
         (cache,e_1,prop1) = Static.elabExpLHS(cache,inEnv,e1,inImpl,false,inPre,info);
         lt = Types.getPropType(prop1);
@@ -5153,7 +5169,7 @@ algorithm
     case (_,Absyn.TUPLE(expressions = expl),e_1,_)
       equation
         true = List.all(expl, AbsynUtil.isCref);
-        failure(Absyn.CALL() = inRhs);
+        failure(Absyn.CALL() = inRhsNoComment);
         s = ExpressionDump.printExpStr(e_1);
         Error.addSourceMessage(Error.TUPLE_ASSIGN_FUNCALL_ONLY, {s}, info);
       then

OMCompiler/Compiler/FrontEnd/Patternm.mo

@@ -310,6 +310,11 @@ algorithm
         (cache,patternTail) = elabPattern(cache,env,tail,tyTail,info);
       then (cache,DAE.PAT_CONS(patternHead,patternTail));
 
+    case (cache,_,Absyn.TUPLE({exp}),_,_,_)
+      algorithm
+        (cache,pattern) := elabPattern2(cache,env,exp,ty,info,numError);
+      then (cache,pattern);
+
     case (cache,_,Absyn.TUPLE(exps),DAE.T_METATUPLE(types = tys),_,_)
       equation
         tys = List.map(tys, Types.boxIfUnboxedType);
@@ -398,6 +403,11 @@ algorithm
 
     case (cache,_,Absyn.CREF(Absyn.WILD()),_,_,_) then (cache,DAE.PAT_WILD());
 
+    case (cache,_,Absyn.EXPRESSIONCOMMENT(),_,_,_)
+      equation
+        (cache,pattern) = elabPattern2(cache,env,inLhs.exp,ty,info,numError);
+      then (cache,pattern);
+
     case (_,_,lhs,_,_,_)
       equation
         true = numError == Error.getNumErrorMessages();
@@ -2133,7 +2143,7 @@ protected function elabResultExp
   output Option<DAE.Type> resType;
 algorithm
   (outCache,outBody,resExp,resultInfo,resType) :=
-  matchcontinue (inCache,inEnv,inBody,exp)
+  matchcontinue (inCache,inEnv,inBody,AbsynUtil.stripCommentExpressions(exp))
     local
       DAE.Exp elabExp;
       DAE.Properties prop;
@@ -3086,6 +3096,10 @@ protected function convertExpToPatterns
   output list<Absyn.Exp> outInputs;
 algorithm
   outInputs := match inExp
+    local
+      Absyn.Exp exp;
+    case Absyn.EXPRESSIONCOMMENT(exp=exp) then convertExpToPatterns(exp);
+    case Absyn.TUPLE({exp}) then convertExpToPatterns(exp);
     case Absyn.TUPLE() then inExp.expressions;
     else {inExp};
   end match;