Add tests for the Modelica diff lexer

openmodelica/bootstrapping/DiffAlgorithm.mos

@@ -5,12 +5,14 @@ echo(false);
 prefixPath := if ""==getEnvironmentVar("OMCOMPILERSOURCES") then "../../../OMCompiler/Compiler/" else getEnvironmentVar("OMCOMPILERSOURCES") + "/";
 echo(true);
 loadFiles({
-  prefixPath + "Util/DiffAlgorithm.mo"
+  prefixPath + "Util/DiffAlgorithm.mo",
+  prefixPath + "Util/Print.mo"
 });
 getErrorString();
 
 loadFile("TestDiffAlgorithm.mo");
 getErrorString();
+setCommandLineOptions("+d=rml");
 generateCode(TestDiffAlgorithm);
 getErrorString();
 TestDiffAlgorithm();
@@ -24,6 +26,7 @@ getErrorString();
 // true
 // ""
 // true
+// true
 // ""
 // Calling diff(
 //   seq1={1, 2, 3},

openmodelica/bootstrapping/DiffAlgorithmModelica.mos

@@ -0,0 +1,143 @@
+// status: correct
+// depends: modelicadiff
+
+setCommandLineOptions("+g=MetaModelica +n=1 +d=noevalfunc");
+echo(false);
+prefixPath := if ""==getEnvironmentVar("OMCOMPILERSOURCES") then "../../../OMCompiler/Compiler/" else getEnvironmentVar("OMCOMPILERSOURCES") + "/";
+echo(true);
+loadFiles({
+  prefixPath + "Lexers/LexerModelicaDiff.mo",
+  prefixPath + "Util/DiffAlgorithm.mo",
+  prefixPath + "Util/Print.mo",
+  prefixPath + "Util/System.mo"
+});
+getErrorString();
+
+loadFile("TestDiffAlgorithm.mo");
+loadFile("TestDiffAlgorithmModelica.mo");
+getErrorString();
+
+// Create some diff files for testing?
+// list(writeFile("modelicadiff/"+) for f in {"4"});
+// runScriptParallel({},useThreads=false);
+
+setCommandLineOptions("+d=noevalfunc,rml");
+
+generateCode(diffModelica);
+getErrorString();
+
+list(diffModelica("modelicadiff/"+String(f)+".before", "modelicadiff/"+String(f)+".after") for f in 1:5);
+getErrorString();
+
+// Result:
+// true
+// true
+// true
+// ""
+// true
+// true
+// ""
+// true
+// true
+// ""
+// modelicadiff/1.before:
+// model M
+// end M;
+//
+//
+// model M
+//   Real r;
+// end M;
+//
+//
+// modelicadiff/2.before:
+// model M
+//   Real x;
+// Real y;
+// end M;
+//
+//
+// model M
+//   Real x;
+// Real y;
+// end M;
+//
+//
+// modelicadiff/3.before:
+// model aa
+//   Real x;
+// algorithm
+//   if time > 0.5 then
+//   //this is a comment
+//     x := 2;
+//   else
+//     x := 1;
+//   end if;
+// end aa;
+//
+//
+// model aa
+//   Real x;
+// algorithm
+//   if time > 0.5 then
+//   //this is a comment
+//     x := 2;
+//   else
+//     x := 1;
+//   end if;
+// end aa;
+//
+//
+// modelicadiff/4.before:
+// model CommentBug " Model"
+//   /* This is a several-line comment
+//      line two
+//      line three
+//      line four
+//   */
+// end CommentBug;
+//
+//
+// model CommentBug " Model"
+//   /* This is a several-line comment
+//      line two
+//      line three
+//      line four
+//   */
+// end CommentBug;
+//
+//
+// modelicadiff/5.before:
+// model aa
+//   Real x;
+// algorithm
+//   if time > 0.5 then
+//   /* this is a comment
+//   with
+//     several
+//   lines
+// */    x := 2;
+//   else
+//     x := 1;
+//   end if;
+// end aa;
+//
+//
+// model aa
+//   Real x;
+// algorithm
+//   if time > 0.5 then
+//   /* this is a comment
+//   with
+//     several
+//   lines
+// */    x := 2;
+//   else
+//     x := 1;
+//   end if;
+// end aa;
+//
+//
+// {,,,,}
+// ""
+// endResult

openmodelica/bootstrapping/Makefile

@@ -3,6 +3,7 @@ TEST = ../../rtest
 
 TESTFILES = \
 DiffAlgorithm.mos \
+DiffAlgorithmModelica.mos \
 DumpTest.mos \
 ExpressionTest.mos \
 GraphTest.mos \

openmodelica/bootstrapping/TestDiffAlgorithm.mo

@@ -9,25 +9,25 @@ protected
   list<tuple<DiffAlgorithm.Diff, list<String>>> strDiffs;
 algorithm
   intDiffs := debug_diff({1,2,3},{1,2,3}, intEq, intString);
-  DiffAlgorithm.printDiffTerminalColor(intDiffs, intString);
+  print(DiffAlgorithm.printDiffTerminalColor(intDiffs, intString));
   print("\n");
   intDiffs := debug_diff({1,2,3},{1,2,3,4,5,6}, intEq, intString);
-  DiffAlgorithm.printDiffTerminalColor(intDiffs, intString);
+  print(DiffAlgorithm.printDiffTerminalColor(intDiffs, intString));
   print("\n");
   intDiffs := debug_diff({1,2,3,4,5,6},{1,2,3}, intEq, intString);
-  DiffAlgorithm.printDiffTerminalColor(intDiffs, intString);
+  print(DiffAlgorithm.printDiffTerminalColor(intDiffs, intString));
   print("\n");
   intDiffs := debug_diff({1,2,3,4,5,6},{1,2,3,6}, intEq, intString);
-  DiffAlgorithm.printDiffTerminalColor(intDiffs, intString);
+  print(DiffAlgorithm.printDiffTerminalColor(intDiffs, intString));
   print("\n");
   intDiffs := debug_diff({1},{2}, intEq, intString);
-  DiffAlgorithm.printDiffTerminalColor(intDiffs, intString);
+  print(DiffAlgorithm.printDiffTerminalColor(intDiffs, intString));
   print("\n");
   intDiffs := debug_diff({1,2,3,4,5,6,7,8,9,10},{1,2,6,4,5,7,3,8,9,10}, intEq, intString);
-  DiffAlgorithm.printDiffTerminalColor(intDiffs, intString);
+  print(DiffAlgorithm.printDiffTerminalColor(intDiffs, intString));
   print("\n");
   strDiffs := debug_diff({"1","2","3"},{"1","2","3"}, stringEq, id);
-  DiffAlgorithm.printDiffTerminalColor(strDiffs, id);
+  print(DiffAlgorithm.printDiffTerminalColor(strDiffs, id));
 end TestDiffAlgorithm;
 
 function debug_diff<T>

openmodelica/bootstrapping/TestDiffAlgorithmModelica.mo

@@ -0,0 +1,174 @@
+function diffModelica
+  import LexerModelicaDiff.{Token,TokenId,tokenContent,scan,scanString};
+  import DiffAlgorithm.{Diff,diff,printActual,printDiffTerminalColor};
+  input String before;
+  input String after;
+protected
+  list<Token> tbefore,tafter,tafter2;
+  list<tuple<Diff, list<Token>>> diffs;
+  String after2;
+algorithm
+  tbefore := scan(before);
+  print(before + ":\n");
+  print(sum(tokenContent(t) for t in tbefore));
+  print("\n\n");
+  tafter := scan(after);
+  diffs := diff(tbefore, tafter, modelicaDiffTokenEq);
+  diffs := filterModelicaDiff(diffs);
+  // Scan a second time, with comments filtered into place
+  after2 := printActual(diffs, LexerModelicaDiff.tokenContent);
+  tafter2 := scanString(after2);
+  diffs := diff(tbefore, tafter2, modelicaDiffTokenEq);
+  diffs := filterModelicaDiff(diffs);
+  print(printDiffTerminalColor(diffs, LexerModelicaDiff.tokenContent));
+  print("\n\n");
+end diffModelica;
+
+function modelicaDiffTokenEq
+  import LexerModelicaDiff.{Token,TokenId,tokenContent};
+  input Token ta,tb;
+  output Boolean b;
+protected
+  LexerModelicaDiff.TokenId ida,idb;
+algorithm
+  LexerModelicaDiff.TOKEN(id=ida) := ta;
+  LexerModelicaDiff.TOKEN(id=idb) := tb;
+  if ida <> idb then
+    b := false;
+    return;
+  end if;
+  b := match ida
+    case TokenId.IDENT then tokenContent(ta)==tokenContent(tb);
+    case TokenId.UNSIGNED_INTEGER then tokenContent(ta)==tokenContent(tb);
+    case TokenId.UNSIGNED_REAL
+      then stringReal(tokenContent(ta))==stringReal(tokenContent(tb));
+    case TokenId.BLOCK_COMMENT
+      then valueEq(blockCommentCanonical(ta),blockCommentCanonical(tb));
+    case TokenId.LINE_COMMENT then tokenContent(ta)==tokenContent(tb);
+    case TokenId.STRING then tokenContent(ta)==tokenContent(tb);
+    case TokenId.WHITESPACE then true; // tokenContent(ta)==tokenContent(tb);
+    else true;
+  end match;
+end modelicaDiffTokenEq;
+
+function filterModelicaDiff
+  import LexerModelicaDiff.{Token,TokenId,tokenContent,TOKEN};
+  import DiffAlgorithm.Diff;
+  input list<tuple<Diff, list<Token>>> diffs;
+  output list<tuple<Diff, list<Token>>> odiffs;
+protected
+  list<String> addedLineComments, removedLineComments;
+  list<list<String>> addedBlockComments, removedBlockComments;
+  list<tuple<Diff, Token>> simpleDiff;
+algorithm
+  // No changes are easy
+  _ := match diffs
+    case {(Diff.Equal,_)}
+      algorithm
+        odiffs := diffs;
+        return;
+      then ();
+    else ();
+  end match;
+
+  odiffs := listReverse(match e
+    local
+      list<Token> ts;
+    case (Diff.Delete,ts as {TOKEN(id=TokenId.WHITESPACE)}) then (Diff.Equal,ts);
+    else e;
+    end match
+
+    for e guard(
+    match e
+      // Single addition of whitespace, not followed by another addition
+      // is suspected garbage added by OMC.
+      case (Diff.Add,{TOKEN(id=TokenId.WHITESPACE)}) then false;
+      case (_,{}) then false;
+      else true;
+    end match
+  ) in diffs);
+
+  // Convert from multiple additions per item to one per item
+  // Costs more memory, but is easier to transform
+  simpleDiff := listAppend(
+    match e
+      local
+        list<Token> ts;
+      case (Diff.Add,ts) then list((Diff.Add,t) for t in ts);
+      case (Diff.Equal,ts) then list((Diff.Equal,t) for t in ts);
+      case (Diff.Delete,ts) then list((Diff.Delete,t) for t in ts);
+    end match
+  for e in odiffs);
+
+  addedLineComments := list(tokenContent(tuple22(e)) for e guard Diff.Add==tuple21(e) and isLineComment(tuple22(e)) in simpleDiff);
+  removedLineComments := list(tokenContent(tuple22(e)) for e guard Diff.Delete==tuple21(e) and isLineComment(tuple22(e)) in simpleDiff);
+
+  addedBlockComments := list(blockCommentCanonical(tuple22(e)) for e guard Diff.Add==tuple21(e) and isBlockComment(tuple22(e)) in simpleDiff);
+  removedBlockComments := list(blockCommentCanonical(tuple22(e)) for e guard Diff.Delete==tuple21(e) and isBlockComment(tuple22(e)) in simpleDiff);
+
+  simpleDiff := list(
+    match e
+      local
+        Token t;
+      case (Diff.Delete,t as TOKEN(id=TokenId.LINE_COMMENT)) then (if listMember(tokenContent(t), addedLineComments) then (Diff.Equal,t) else e);
+      case (Diff.Delete,t as TOKEN(id=TokenId.BLOCK_COMMENT)) then (if listMember(blockCommentCanonical(t), addedBlockComments) then (Diff.Equal,t) else e);
+      else e;
+    end match
+    for e guard(
+    match e
+      local
+        Token t;
+      case (Diff.Add,t as TOKEN(id=TokenId.LINE_COMMENT)) then not listMember(tokenContent(t), removedLineComments);
+      case (Diff.Add,t as TOKEN(id=TokenId.BLOCK_COMMENT)) then not listMember(blockCommentCanonical(t), removedBlockComments);
+      else true;
+    end match
+  ) in simpleDiff);
+
+  odiffs := list(
+    match e
+      local
+        Diff d;
+        Token t;
+      case (d,t) then (d,{t});
+    end match
+    for e in simpleDiff);
+end filterModelicaDiff;
+
+function isBlockComment
+  import LexerModelicaDiff.{Token,TokenId,TOKEN};
+  input Token t;
+  output Boolean b;
+algorithm
+  b := match t case TOKEN(id=TokenId.BLOCK_COMMENT) then true; else false; end match;
+end isBlockComment;
+
+function isLineComment
+  import LexerModelicaDiff.{Token,TokenId,TOKEN};
+  input Token t;
+  output Boolean b;
+algorithm
+  b := match t case TOKEN(id=TokenId.LINE_COMMENT) then true; else false; end match;
+end isLineComment;
+
+function tuple21<A,B>
+  input tuple<A,B> t;
+  output A a;
+algorithm
+  (a,_) := t;
+end tuple21;
+
+function tuple22<A,B>
+  input tuple<A,B> t;
+  output B b;
+algorithm
+  (_,b) := t;
+end tuple22;
+
+function blockCommentCanonical
+  import LexerModelicaDiff.{Token,tokenContent};
+  input Token t;
+  output list<String> lines;
+algorithm
+  // Canonical representation trims whitespace from each line
+  lines := list(System.trim(s) for s in System.strtok(tokenContent(t),"\n"));
+end blockCommentCanonical;

openmodelica/bootstrapping/modelicadiff/1.after

@@ -0,0 +1,3 @@
+model M
+  Real r;
+end M;

openmodelica/bootstrapping/modelicadiff/1.before

@@ -0,0 +1,2 @@
+model M
+end M;

openmodelica/bootstrapping/modelicadiff/2.after

@@ -0,0 +1,4 @@
+model M
+  Real x;
+  Real y;
+end M;