Skip to content

Commit

Permalink
Updated the model testing script to include the possibility of compar…
Browse files Browse the repository at this point in the history 
…ing simulation results

git-svn-id: https://openmodelica.org/svn/OpenModelica/trunk@17671 f25d12d1-65f4-0310-ae8a-bbce733d8d8e
  • Loading branch information
@sjoelund
sjoelund committed Oct 14, 2013
1 parent 62f2dba commit 36222c6
Show file tree
Hide file tree
Showing 5 changed files with 143 additions and 66 deletions.
6 changes: 4 additions & 2 deletions Compiler/FrontEnd/ModelicaBuiltin.mo
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1805,7 +1805,9 @@ annotation(preferredView="text");
end listVariables;

function strtok "Splits the strings at the places given by the token, for example:
strtok(\"abcbdef\",\"b\") => {\"a\",\"c\",\"def\"}"
strtok(\"abcbdef\",\"b\") => {\"a\",\"c\",\"def\"}
strtok(\"abcbdef\",\"cd\") => {\"ab\",\"ef\"}
"
input String string;
input String token;
output String[:] strings;
Expand Down Expand Up @@ -2344,7 +2346,7 @@ public function diffSimulationResults "compares simulation results."
external "builtin";
annotation(Documentation(info="<html>
<p>Takes two result files and compares them. By default, all selected variables that are not equal in the two files are output to diffPrefix.varName.csv.</p>
<p>The output is the generated filesnames</p>
<p>The output is the names of the variables for which files were generated.</p>
</html>"),preferredView="text");
end diffSimulationResults;

Expand Down
31 changes: 31 additions & 0 deletions Compiler/runtime/SimulationResults.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,37 @@ static void* SimulationResultsImpl__readVars(const char *filename, SimulationRes
}
}

static void* SimulationResultsImpl__readVarsFilterAliases(const char *filename, SimulationResult_Globals* simresglob)
{
const char *msg[2] = {"",""};
if (UNKNOWN_PLOT == SimulationResultsImpl__openFile(filename,simresglob)) {
return mk_nil();
}
switch (simresglob->curFormat) {
case MATLAB4: {
void *res = mk_nil();
int i;
int *params = (int*) calloc(simresglob->matReader.nparam+1,sizeof(int));
int *vars = (int*) calloc(simresglob->matReader.nvar+1,sizeof(int));
for (i=simresglob->matReader.nall-1; i>=0; i--) {
if (0 >= simresglob->matReader.allInfo[i].index) continue; /* Negated aliases always have a real variable, so skip it */
if (simresglob->matReader.allInfo[i].isParam && params[simresglob->matReader.allInfo[i].index]) continue;
if (!simresglob->matReader.allInfo[i].isParam && vars[simresglob->matReader.allInfo[i].index]) continue;
if (simresglob->matReader.allInfo[i].isParam) {
params[simresglob->matReader.allInfo[i].index] = 1;
} else {
vars[simresglob->matReader.allInfo[i].index] = 1;
}
res = mk_cons(mk_scon(simresglob->matReader.allInfo[i].name),res);
}
free(params);
free(vars);
return res;
}
default: return SimulationResultsImpl__readVars(filename, simresglob);
}
}

static void* SimulationResultsImpl__readDataset(const char *filename, void *vars, int dimsize, SimulationResult_Globals* simresglob)
{
const char *msg[2] = {"",""};
Expand Down
38 changes: 17 additions & 21 deletions Compiler/runtime/SimulationResultsCmp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ static inline void fixCommaInName(char **str, size_t len)

if (nc > 0) {

newvar = (char*) malloc(len+nc+1);
newvar = (char*) malloc(len+nc+10);
k = 0;
for (j=0;j<len;j++) {
newvar[k] = (*str)[j];
Expand Down Expand Up @@ -237,7 +237,7 @@ static char AlmostEqualRelativeAndAbs(double A, double B)
return 0;
}

static unsigned int cmpData(int isResultCmp, char* varname, DataField *time, DataField *reftime, DataField *data, DataField *refdata, double reltol, double abstol, DiffDataField *ddf, char **cmpdiffvars, unsigned int vardiffindx, int keepEqualResults, void **diffLst, char *prefix)
static unsigned int cmpData(int isResultCmp, char* varname, DataField *time, DataField *reftime, DataField *data, DataField *refdata, double reltol, double abstol, DiffDataField *ddf, char **cmpdiffvars, unsigned int vardiffindx, int keepEqualResults, void **diffLst, const char *prefix)
{
unsigned int i,j,k,j_event;
double t,tr,d,dr,err,d_left,d_right,dr_left,dr_right,t_event;
Expand All @@ -249,11 +249,11 @@ static unsigned int cmpData(int isResultCmp, char* varname, DataField *time, Dat
FILE *fout = NULL;
char *fname = NULL;
if (!isResultCmp) {
fname = (char*) malloc(15 + strlen(varname));
fname = (char*) malloc(25 + strlen(prefix) + strlen(varname));
sprintf(fname, "%s.%s.csv", prefix, varname);
fout = fopen(fname,"w");
if (fout) {
fprintf(fout, "time,reference,actual,err,relerr,\"%s\"\n",varname);
fprintf(fout, "time,reference,actual,err,relerr,threshold,\"%s\"\n",varname);
}
}
for (i=0;i<refdata->n;i++){
Expand Down Expand Up @@ -382,7 +382,7 @@ static unsigned int cmpData(int isResultCmp, char* varname, DataField *time, Dat
#ifdef DEBUGOUTPUT
fprintf(stderr, "delta:%.6g reltol:%.6g\n",err,average);
#endif
if ( err < average){
if ( err < average ) {
continue;
}
}
Expand Down Expand Up @@ -515,7 +515,7 @@ static unsigned int cmpData(int isResultCmp, char* varname, DataField *time, Dat
fprintf(stderr, "delta:%.6g reltol:%.6g\n",err,average);
#endif
if (fout) {
fprintf(fout, "%.6g,%.6g,%.6g,%.6g,%.6g\n",tr,d,dr,err,AlmostEqualRelativeAndAbs(d,0) ? err/average : absdouble(err/d));
fprintf(fout, "%.6g,%.6g,%.6g,%.6g,%.6g,%.6g\n",tr,d,dr,err,AlmostEqualRelativeAndAbs(d,0) ? err/average : absdouble(err/d),average);
}
if ( err > average){
if (j+1<reftime->n) {
Expand Down Expand Up @@ -552,7 +552,7 @@ static unsigned int cmpData(int isResultCmp, char* varname, DataField *time, Dat
cmpdiffvars[vardiffindx] = varname;
vardiffindx++;
if (!isResultCmp) {
*diffLst = mk_cons(mk_scon(fname),*diffLst);
*diffLst = mk_cons(mk_scon(varname),*diffLst);
}
}
if (fout) {
Expand Down Expand Up @@ -613,7 +613,7 @@ void* SimulationResultsCmp_compareResults(int isResultCmp, int runningTestsuite,
unsigned int ncmpvars = 0;
unsigned int ngetfailedvars = 0;
void *allvars,*allvarsref,*res;
unsigned int i,size,size_ref,len,oldlen,j,k;
unsigned int i,size,size_ref,len,j,k;
char *var,*var1,*var2;
DataField time,timeref,data,dataref;
DiffDataField ddf;
Expand All @@ -622,13 +622,12 @@ void* SimulationResultsCmp_compareResults(int isResultCmp, int runningTestsuite,
ddf.data=NULL;
ddf.n=0;
ddf.n_max=0;
oldlen = 0;
len = 1;

/* open files */
/* fprintf(stderr, "Open File %s\n", filename); */
if (UNKNOWN_PLOT == SimulationResultsImpl__openFile(filename,&simresglob_c)) {
char *str = (char*) malloc(21+strlen(filename));
char *str = (char*) malloc(25+strlen(filename));
*str = 0;
strcat(strcat(str,"Error opening file: "),runningTestsuite ? SystemImpl__basename(filename) : filename);
void *res = mk_scon(str);
Expand All @@ -637,7 +636,7 @@ void* SimulationResultsCmp_compareResults(int isResultCmp, int runningTestsuite,
}
/* fprintf(stderr, "Open File %s\n", reffilename); */
if (UNKNOWN_PLOT == SimulationResultsImpl__openFile(reffilename,&simresglob_ref)) {
char *str = (char*) malloc(31+strlen(reffilename));
char *str = (char*) malloc(35+strlen(reffilename));
*str = 0;
strcat(strcat(str,"Error opening reference file: "),runningTestsuite ? SystemImpl__basename(reffilename) : reffilename);
void *res = mk_scon(str);
Expand All @@ -653,8 +652,8 @@ void* SimulationResultsCmp_compareResults(int isResultCmp, int runningTestsuite,
/* get vars to compare */
cmpvars = getVars(vars,&ncmpvars);
/* if no var compare all vars */
allvars = SimulationResultsImpl__readVars(filename,&simresglob_c);
allvarsref = SimulationResultsImpl__readVars(reffilename,&simresglob_ref);
allvars = SimulationResultsImpl__readVarsFilterAliases(filename,&simresglob_c);
allvarsref = SimulationResultsImpl__readVarsFilterAliases(reffilename,&simresglob_ref);
if (ncmpvars==0){
cmpvars = getVars(allvarsref,&ncmpvars);
if (ncmpvars==0) return mk_cons(mk_scon("Error Get Vars!"),mk_nil());
Expand Down Expand Up @@ -697,11 +696,8 @@ void* SimulationResultsCmp_compareResults(int isResultCmp, int runningTestsuite,
for (i=0;i<ncmpvars;i++) {
var = cmpvars[i];
len = strlen(var);
if (oldlen < len) {
if (var1) free(var1);
var1 = (char*) malloc(len+1);
oldlen = len;
}
if (var1) free(var1);
var1 = (char*) malloc(len+10);
k = 0;
for (j=0;j<len;j++) {
if (var[j] !='\"' ) {
Expand All @@ -715,7 +711,7 @@ void* SimulationResultsCmp_compareResults(int isResultCmp, int runningTestsuite,
dataref = getData(var1,reffilename,size_ref,&simresglob_ref);
if (dataref.n==0) {
if (var2) free(var2);
var2 = (char*) malloc(len+1);
var2 = (char*) malloc(len+10);
strncpy(var2,var1,len+1);
fixDerInName(var2,len);
fixCommaInName(&var2,len);
Expand Down Expand Up @@ -773,8 +769,8 @@ void* SimulationResultsCmp_compareResults(int isResultCmp, int runningTestsuite,
*success = 0==vardiffindx;
}

if (var1) free(var1);
if (var2) free(var2);
// if (var1) free(var1);
// if (var2) free(var2);
if (ddf.data) free(ddf.data);
if (cmpvars) free(cmpvars);
if (time.data) free(time.data);
Expand Down
122 changes: 84 additions & 38 deletions Examples/BuildModelRecursive.mos
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

library:=$TypeName(Modelica);
libraryVersion:="default";
referenceFiles:="";

setCommandLineOptions({"+g=Modelica","+d=nogen,"});
OpenModelica.Scripting.Internal.Time.timerTick(OpenModelica.Scripting.Internal.Time.RT_CLOCK_USER_RESERVED);
Expand All @@ -28,7 +29,8 @@ getErrorString();
// writeFile("x",sum(s + "\n" for s in a));

print("Number of classes to build: " + String(size(a,1)));
system("rm -f "+libraryString+"* " + log);
system("rm -rf files/ "+libraryString+"* " + log);
mkdir("files");

/*** This is the stuff executed for each model ***/

Expand All @@ -43,6 +45,47 @@ writeFile(statFile + \".compile\", \"killed\");getErrorString();
writeFile(statFile + \".sim\", \"killed\");getErrorString();
"+loadModelCommand+"
setCommandLineOptions(\"+d=nogen,initialization\"); setMatchingAlgorithm(\"PFPlusExt\"); setIndexReductionMethod(\"dynamicStateSelection\");
loadString(\"
function cell
input Real time;
input Boolean greenOnOk := false;
input String greenColor := \\\"#00FF00\\\";
output String str;
algorithm
str := if time==-1.0 then \\\"<td bgcolor=\\\\\\\"#FF0000\\\\\\\">&nbsp;</td>\\\" else ((if greenOnOk then \\\"<td bgcolor=\\\\\\\"\\\" + greenColor + \\\"\\\\\\\">\\\" else \\\"<td>\\\") +
OpenModelica.Scripting.Internal.Time.readableTime(if time < 1e-5 then 0 else time)+\\\"</td>\\\");
end cell;
function length input String str; output Integer len; external \\\"C\\\" len=ModelicaStrings_length(str); end length;
function substring input String str; input Integer i1,i2; output String out; external \\\"C\\\" out=ModelicaStrings_substring(str,i1,i2); end substring;
function firstPart input String str; input Integer maxLength := 4096; output String out; algorithm out := if length(str) < maxLength then str else (substring(str,1,maxLength) + \\\"\n... (output truncated, was \\\" + String(length(str)) + \\\" bytes)\\\"); end firstPart;
function csvFileToVariable
input String str;
output String out;
protected
String matches[2];
algorithm
(,matches) := OpenModelica.Scripting.regex(str,\\\"^.*[.]diff[.](.*)[.]csv$\\\",2);
out := matches[2];
end csvFileToVariable;
function testcase
input Real time;
input Boolean success;
input String errXMLLong;
output String str;
protected
String errXML;
algorithm
errXML := firstPart(errXMLLong,16*1024); // 16 kB log per test should be enough; increase if there is a good reason to do so
str:=\\\"<testcase name=\\\\\\\""+s+"\\\\\\\" time=\\\\\\\"\\\"+String(time)+\\\"\\\\\\\">\\\"+(
if not success then \\\"<error type=\\\\\\\"Failure\\\\\\\" />\\\"
else \\\"\\\"
)+(
if errXML <> \\\"\\\" then (\\\"<system-out>\\\" + errXML + \\\"</system-out>\\\")
else \\\"\\\"
)+
\\\"</testcase>\n\\\";
end testcase;
\");getErrorString();
res:=buildModel("+s+");
greenColor := \"#00FF00\";
if res[1] == \"\" then
Expand All @@ -51,8 +94,8 @@ if res[1] == \"\" then
res:=buildModel("+s+");
greenColor := \"#FFFF00\";
end if;
errFile:=\""+s+".err\";
simFile:=\""+s+".sim\";
errFile:=\"files/"+s+".err\";
simFile:=\"files/"+s+".sim\";
err:=getErrorString();
system(\"rm -f \" + errFile);
writeFile(simFile,\"\");
Expand All @@ -75,46 +118,48 @@ OpenModelica.Scripting.Internal.Time.timerTick(OpenModelica.Scripting.Internal.T
buildRes := res[1] <> \"\";
simRes := if not buildRes then false else 0 == system(\"ulimit -t "+ulimitExe+" ; ./"+s+" > \"+simFile+\" 2>&1\");
timeSim := OpenModelica.Scripting.Internal.Time.timerTock(OpenModelica.Scripting.Internal.Time.RT_CLOCK_USER_RESERVED);
referenceExists := false;
referenceCell := \"<td bgcolor=\\\"#FFFF00\\\">&nbsp;</td>\";
if simRes then
system(\"touch "+s+".simsuccess\");
reference := \""+referenceFiles+"/"+s+".mat\";
referenceExists := regularFileExists(reference);
prefix := \"files/"+s+".diff\";
if referenceExists then
OpenModelica.Scripting.Internal.Time.timerTick(OpenModelica.Scripting.Internal.Time.RT_CLOCK_USER_RESERVED);
(referenceOK,diffVars) := diffSimulationResults(\""+s+"_res.mat\",reference,prefix);
timeDiff := OpenModelica.Scripting.Internal.Time.timerTock(OpenModelica.Scripting.Internal.Time.RT_CLOCK_USER_RESERVED);
diffFiles := {prefix + \".\" + var + \".csv\" for var in diffVars};
if not referenceOK then
writeFile(\""+s+".gnuplot\",\"set term svg size 600,800
set datafile separator \\\",\\\"
set format y \\\"%g\\\"
set samples 300
\" + sum(\"set output \\\"\"+file+\".svg\\\"
set multiplot layout 3,1
set title \\\"Result difference of \"+csvFileToVariable(file)+\"\\\"
plot \\\"\"+file+\"\\\" using \\\"time\\\":\\\"reference\\\" smooth csplines with lines, \\\"\"+file+\"\\\" using \\\"time\\\":\\\"actual\\\" smooth csplines with lines
unset title
plot \\\"\"+file+\"\\\" using \\\"time\\\":\\\"err\\\" title \\\"absolute error\\\" smooth unique with lines, \\\"\"+file+\"\\\" using \\\"time\\\":\\\"threshold\\\" smooth unique with lines
plot \\\"\"+file+\"\\\" using \\\"time\\\":\\\"relerr\\\" title \\\"relative error\\\" smooth unique with lines
unset multiplot
\" for file in diffFiles));
system(\"gnuplot '"+s+".gnuplot'\");
timeDiff := OpenModelica.Scripting.Internal.Time.timerTock(OpenModelica.Scripting.Internal.Time.RT_CLOCK_USER_RESERVED);
referenceCell := \"<td bgcolor=\\\"#FF0000\\\">\"+OpenModelica.Scripting.Internal.Time.readableTime(timeDiff)+\", <a href=\\\"files/"+s+".diff.html\\\">\"+String(size(diffFiles,1))+\" failed</a></td>\";
writeFile(\"files/"+s+".diff.html\",\"<html><body><h1>"+s+" differences from the reference file</h1><ul>\" + sum(\"<li>\"+csvFileToVariable(file)+\" <a href=\\\"\"+OpenModelica.Scripting.basename(file)+\".svg\\\">(plot)</a> <a href=\\\"\"+OpenModelica.Scripting.basename(file)+\"\\\">(csv)</a></li>\" for file in diffFiles) + \"</ul></body></html>\");
else
referenceCell := \"<td bgcolor=\\\"#00FF00\\\">\"+OpenModelica.Scripting.Internal.Time.readableTime(timeDiff)+\"</td>\";
end if;
end if;
else
referenceCell := \"<td bgcolor=\\\"#FF0000\\\">&nbsp;</td>\";
end if;

loadString(\"
function cell
input Real time;
input Boolean greenOnOk := false;
input String greenColor := \\\"#00FF00\\\";
output String str;
algorithm
str := if time==-1.0 then \\\"<td bgcolor=\\\\\\\"#FF0000\\\\\\\">&nbsp;</td>\\\" else ((if greenOnOk then \\\"<td bgcolor=\\\\\\\"\\\" + greenColor + \\\"\\\\\\\">\\\" else \\\"<td>\\\") +
OpenModelica.Scripting.Internal.Time.readableTime(if time < 1e-5 then 0 else time)+\\\"</td>\\\");
end cell;
function length input String str; output Integer len; external \\\"C\\\" len=ModelicaStrings_length(str); end length;
function substring input String str; input Integer i1,i2; output String out; external \\\"C\\\" out=ModelicaStrings_substring(str,i1,i2); end substring;
function firstPart input String str; input Integer maxLength := 4096; output String out; algorithm out := if length(str) < maxLength then str else (substring(str,1,maxLength) + \\\"\n... (output truncated, was \\\" + String(length(str)) + \\\" bytes)\\\"); end firstPart;
function testcase
input Real time;
input Boolean success;
input String errXMLLong;
output String str;
protected
String errXML;
algorithm
errXML := firstPart(errXMLLong,16*1024); // 16 kB log per test should be enough; increase if there is a good reason to do so
str:=\\\"<testcase name=\\\\\\\""+s+"\\\\\\\" time=\\\\\\\"\\\"+String(time)+\\\"\\\\\\\">\\\"+(
if not success then \\\"<error type=\\\\\\\"Failure\\\\\\\" />\\\"
else \\\"\\\"
)+(
if errXML <> \\\"\\\" then (\\\"<system-out>\\\" + errXML + \\\"</system-out>\\\")
else \\\"\\\"
)+
\\\"</testcase>\n\\\";
end testcase;
\");getErrorString();
sim:=readFile(simFile);
str:=\"<tr><td>\" + (if err <> \"\" then \"<a href=\"+errFile+\">"+s+"</a>\" else \""+s+"\") +
(if sim <> \"\" then \" (<a href=\"+simFile+\">sim</a>)\" else \"\") +
\"</td>\" +
\"</td>\" + referenceCell +
cell(if simRes then timeSim else -1.0,greenOnOk=true,greenColor=greenColor) +
\"<td bgcolor=\\\"\"+(if res[1]<>\"\" then greenColor else \"#FF0000\")+\"\\\">\"+OpenModelica.Scripting.Internal.Time.readableTime(total)+\"</td>\" +
sum(cell(d,greenColor=greenColor) for d in {frontend,backend,simcode,templates,build}) + \"</tr>\\n\";getErrorString();
Expand Down Expand Up @@ -148,10 +193,11 @@ str:="<h1>Recursive BuildModel Test</h1>
<p>Links are provided if getErrorString() or the simulation generates output. <font style=\"background-color:#00FF00\">Green</font> means success, <font style=\"background-color:#FFFF00\">yellow</font> means success if +d=scodeInstShortcut was used. <font style=\"background-color:#FF0000\">Red</font> is bad and in general signifies a failure.</p>
";
writeFile(log,"<html><head><title>BuildModel Results</title></head><body>" + str);
writeFile(log,"<table><tr><th>Model</th><th>Simulate</th><th>Total buildModel</th><th>Frontend</th><th>Backend</th><th>SimCode</th><th>Templates</th><th>Compile</th></tr>\n",append=true);getErrorString();
writeFile(log,"<table><tr><th>Model</th><th>Verified</th><th>Simulate</th><th>Total buildModel</th><th>Frontend</th><th>Backend</th><th>SimCode</th><th>Templates</th><th>Compile</th></tr>\n",append=true);getErrorString();
writeFile(log,sum(readFile(s + ".stat") + "\n" for s in a),append=true);getErrorString();
writeFile(log,"</table></body></html>",append=true);getErrorString();
writeFile(log,"</table><hr /><p><a href=\"BuildModelRecursive.tar.xz\">Offline version</a></p></body></html>",append=true);getErrorString();

{writeFile("BuildModelRecursive." + phase + ".xml", "<testsuite>\n" + sum(readFile(s + ".stat." + phase) for s in a) + "</testsuite>\n") for phase in {"flat","compile","sim"}};getErrorString();

system("rm -f tidy.err; for f in *.err; do (grep -v 'Duplicate modification of element' $f | sed s,^.*omlibrary/,[, | grep -v 'Error building simulator' | grep -v 'has no value, and is fixed during initialization' | grep -5E '(Warning|Error|Notification):' > tmp.tidy && test -s tmp.tidy && (basename $f && cat tmp.tidy)) >> tidy.err; done");
system("tar cJf BuildModelRecursive.tar.xz BuildModelRecursive.html files tidy.err");

0 comments on commit 36222c6

Please sign in to comment.