More options for ModelicaCompliance testing

- Added the ability to add custom flags (to test newInst)
- Optionally add a list of tests expected to fail (for Jenkins)

Belonging to [master]:
  - #2571

Author: sjoelund

Examples/ComplianceSuite.mos

@@ -1,15 +1,20 @@
 // Runs the ModelicaCompliance suite
-// Output is the files "openmodelica.xml" and "openmodelica.html", which is a valid jUnit testcase and possible to generate html reports from (the generated file is one example, using the default xsl)
+// Default output is the files "openmodelica.xml" and "openmodelica.html", which is a valid jUnit testcase and possible to generate html reports from (the generated file is one example, using the default xsl)
+// A file called "openmodelica.ignore.xml" is also generated, which can be used together with a file to mark skipped tests (they still run to see if the results have improved)
+// COMPLIANCEEXTRAOMFLAGS=-d=newInst COMPLIANCEEXTRAREPORTFLAGS=--prefix=openmodelica-newinst can be used as well
 
 setCommandLineOptions("-g=Modelica");
 loadModel(ModelicaCompliance);getErrorString();
 setCommandLineOptions("-g=MetaModelica");
 writeFile("version","omc "+getVersion()+" compliance suite "+getVersion(ModelicaCompliance));getErrorString();
 
+COMPLIANCEEXTRAOMFLAGS := getEnvironmentVar("COMPLIANCEEXTRAFLAGS");
+COMPLIANCEEXTRAREPORTFLAGS := getEnvironmentVar("COMPLIANCEEXTRAREPORTFLAGS");
+
 loadString("
 function last
   input String str[:];
-  output String ostr := str[end];
+  output String ostr = str[end];
 end last;
 function fixName
   input String str;
@@ -23,12 +28,12 @@ system("rm -f ModelicaCompliance.*.res");getErrorString();
 
 echo(false);
 allClasses:={cl for cl guard classAnnotationExists(cl,__ModelicaAssociation.TestCase.shouldPass) in getClassNames(recursive=true)};getErrorString();
-commands:={"ulimit -t 120 && ("+getInstallationDirectoryPath()+"/bin/omc -d=nogen \"" + fixName(OpenModelica.Scripting.typeNameString(cl)) + ".mos\"  > /dev/null 2>&1)" for cl in allClasses};
+commands:={"ulimit -t 120 && ("+getInstallationDirectoryPath()+"/bin/omc -d=nogen "+COMPLIANCEEXTRAOMFLAGS+" \"" + fixName(OpenModelica.Scripting.typeNameString(cl)) + ".mos\"  > /dev/null 2>&1)" for cl in allClasses};
 echo(true);
 String(size(allClasses,1)) + " number of classes";
 
 min(OpenModelica.Scripting.writeFile(fixName(typeNameString(cl))+".mos","
-OpenModelica.Scripting.writeFile(\""+fixName(typeNameString(cl))+".res\",\"<testcase name=\\\""+fixName(last(OpenModelica.Scripting.typeNameStrings(cl)))+"\\\"><error type=\\\"killed\\\" message=\\\"killed or crashed\\\"/></testcase>\");
+OpenModelica.Scripting.writeFile(\""+fixName(typeNameString(cl))+".res\",\"{\\\"killed\\\":true}\");
 
 loadModel(ModelicaCompliance);getErrorString();
 // delete classes that generate errors all the time
@@ -59,27 +64,14 @@ success := if shouldPass then resultFile<>\"\" else resultFile==\"\";
 messages := rec.messages;
 messages := messages + err;
 messages := messages + (if not success and not shouldPass /* This is correct; try to figure out why (hint: success is a poor name) */ then \"\nSucceeded, but expected failure\" else \"\");
-messages := OpenModelica.Scripting.escapeXML(firstPart(messages));
+messages := stringReplace(stringReplace(firstPart(messages),\"\\\"\",\"\\\\\\\"\"),\"\n\",\"\\\\n\");
 errorType := if shouldPass then \"failed\" else \"expected failure\";
-writeFile(\""+fixName(typeNameString(cl))+".res\",\"<testcase name=\\\""+fixName(last(OpenModelica.Scripting.typeNameStrings(cl)))+"\\\">\"+(if not success then \"<error type=\\\"\"+errorType+\"\\\" message=\\\"\"+messages+\"\\\"/>\" else \"\")+\"</testcase>\n\");
+time := rec.timeTotal;
+writeFile(\""+fixName(typeNameString(cl))+".res\",\"{\\\"time\\\":\"+String(time)+\", \\\"success\\\":\"+String(success)+\", \\\"messages\\\":\\\"\"+messages+\"\\\", \\\"shouldPass\\\":\"+String(shouldPass)+\"}\n\");
 ") for cl in allClasses);
 getErrorString();
-system_parallel(commands);getErrorString();
 
-writeFile("ComplianceSuite.sh","#!/bin/bash
-OMCRESULT=openmodelica.xml
-echo '<?xml version=\"1.0\" encoding=\"UTF-8\"?>' > \"$OMCRESULT\"
-echo '<testsuites name=\"'`cat version`'\">' >> \"$OMCRESULT\"
-for f in `ls ModelicaCompliance.*.res | sed 's/[.][^.]*[.]res$//' | sort -u`; do
-  echo \"<testsuite name=\\\"$f\\\">\" >> \"$OMCRESULT\"
-  for t in `ls \"$f\".*.res | grep \"$f[.][^.]*[.]res\"`; do
-    iconv -f utf8 -t utf8 -c \"$t\" >> \"$OMCRESULT\"
-  done
-  echo \"</testsuite>\" >> \"$OMCRESULT\"
-done
-echo '</testsuites>' >> \"$OMCRESULT\"
-echo \"$OMCRESULT\"
-");getErrorString();
-system("bash ComplianceSuite.sh");
 filename := OpenModelica.Scripting.uriToFilename("modelica://ModelicaCompliance/Resources/tools/xml-report/report.xsl");getErrorString();
+system_parallel(commands);getErrorString();
+system("python3 "+getInstallationDirectoryPath()+"/share/doc/omc/testmodels/ComplianceSuite.py --version=\"`cat version`\"" + COMPLIANCEEXTRAREPORTFLAGS);getErrorString();
 system("xsltproc -o openmodelica.html '" + filename + "' openmodelica.xml");getErrorString();

Examples/ComplianceSuite.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+import argparse
+from junit_xml import TestSuite, TestCase
+import glob
+import simplejson
+import subprocess
+import time
+
+def readTest(f, expectedFailures):
+  cl = ".".join(f.split(".")[:-1])
+  name = f.split(".")[-2]
+  with open(f) as fin:
+    res = simplejson.load(fin)
+  if "killed" in res:
+    tc = TestCase(name, cl, 0, '', '')
+    tc.add_error_info('Killed or crashed')
+    return (tc, tc, cl)
+
+  tc1 = TestCase(name, cl, res["time"], res["messages"], '')
+  tc2 = TestCase(name, cl, res["time"], res["messages"], '')
+  success = res["success"]
+  shouldPass = res["shouldPass"]
+  expectFail = cl in expectedFailures
+  if expectFail:
+    if success:
+      tc1.add_error_info('This testcase started working (failure was expected)')
+    else:
+      tc1.add_skipped_info('This testcase still fails (as expected)')
+  elif not success:
+    if shouldPass:
+      tc1.add_error_info('failed')
+    else:
+      tc1.add_error_info('expected failure, but passed')
+  if not success:
+    if shouldPass:
+      tc2.add_error_info('failed')
+    else:
+      tc2.add_error_info('expected failure, but passed')
+  return (tc1, tc2, None if success else cl)
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser(description='OpenModelica ModelicaCompliance testing tool')
+  parser.add_argument('--expectedFailures', default=None)
+  parser.add_argument('--outPrefix', default="openmodelica")
+  parser.add_argument('--version', default="OpenModelica ???")
+  args = parser.parse_args()
+  expectedFailuresFile = args.expectedFailures
+  outPrefix = args.outPrefix
+  version = args.version
+
+  expectedFailures = set()
+  if expectedFailuresFile:
+    with open(expectedFailuresFile) as fin:
+      expectedFailures = set(l.strip() for l in fin.readlines())
+  print(expectedFailures)
+
+  res = [readTest(f, expectedFailures) for f in glob.glob("*.res")]
+
+  (tcs1,tcs2,failures) = zip(*res)
+  ts1 = TestSuite(version, tcs1)
+  ts2 = TestSuite(version, tcs2)
+  if outPrefix:
+    with open("%s.ignore.xml" % outPrefix, "w") as fout:
+      TestSuite.to_file(fout, [ts1], prettyprint=False)
+    with open("%s.xml" % outPrefix, "w") as fout:
+      TestSuite.to_file(fout, [ts2], prettyprint=False)
+    with open("%s.failures" % outPrefix, "w") as fout:
+      for fail in failures:
+        if fail:
+          fout.write(fail + "\n")