Added threading support for loadFile(), called loadFiles(). Saves rou…

…ghly 1/3 the time of loading the compiler sources using 5 threads.

git-svn-id: https://openmodelica.org/svn/OpenModelica/trunk@18634 f25d12d1-65f4-0310-ae8a-bbce733d8d8e

Compiler/FrontEnd/ClassLoader.mo

@@ -338,7 +338,7 @@ algorithm
         checkOnLoadMessage(p1);
       then p1;
 
-    // faliling
+    // failing
     else
       equation
         Debug.fprint(Flags.FAILTRACE, "ClassLoader.loadFile failed: "+&name+&"\n");

Compiler/FrontEnd/ModelicaBuiltin.mo

@@ -916,6 +916,15 @@ external "builtin";
 annotation(preferredView="text");
 end loadFile;
 
+function loadFiles "load files (*.mo) and merges them with the loaded AST."
+  input String[:] fileNames;
+  input String encoding := "UTF-8";
+  input Integer numThreads := OpenModelica.Scripting.numProcessors();
+  output Boolean success;
+external "builtin";
+annotation(preferredView="text");
+end loadFiles;
+
 function loadString "Parses the data and merges the resulting AST with ithe
   loaded AST.
   If a filename is given, it is used to provide error-messages as if the string

Compiler/FrontEnd/Parser.mo

@@ -52,8 +52,11 @@ public function parse "Parse a mo-file"
   input String filename;
   input String encoding;
   output Absyn.Program outProgram;
+protected
+  String realpath;
 algorithm
-  outProgram := ParserExt.parse(System.realpath(filename), Util.testsuiteFriendly(System.realpath(filename)), Config.acceptedGrammar(), encoding, Flags.getConfigEnum(Flags.LANGUAGE_STANDARD), Config.getRunningTestsuite());
+  realpath := System.realpath(filename);
+  outProgram := ParserExt.parse(realpath, Util.testsuiteFriendly(System.realpath(filename)), Config.acceptedGrammar(), encoding, Flags.getConfigEnum(Flags.LANGUAGE_STANDARD), Config.getRunningTestsuite());
   /* Check that the program is not totally off the charts */
   _ := SCodeUtil.translateAbsyn2SCode(outProgram);
 end parse;

Compiler/Script/CevalScript.mo

@@ -840,6 +840,7 @@ algorithm
       list<Env.Frame> env;
       GlobalScript.SymbolTable newst,st_1,st;
       Absyn.Program p,pnew,newp,ptot;
+      list<Absyn.Program> newps;
       list<GlobalScript.InstantiatedClass> ic,ic_1;
       list<GlobalScript.Variable> iv;
       list<GlobalScript.CompiledCFunction> cf;
@@ -2091,6 +2092,25 @@ algorithm
     case (cache,env,"loadFile",_,st,_)
       then (cache,Values.BOOL(false),st);
 
+    case (cache,env,"loadFiles",{Values.ARRAY(valueLst=vals),Values.STRING(encoding),Values.INTEGER(i)},
+          (st as GlobalScript.SYMBOLTABLE(
+            ast = p,depends=aDep,instClsLst = ic,
+            lstVarVal = iv,compiledFunctions = cf,
+            loadedFiles = lf)),_)
+      equation
+        strs = List.mapMap(vals,ValuesUtil.extractValueString,Util.testsuiteFriendlyPath);
+        System.GC_disable();
+        newps = System.launchParallelTasks(i, List.map1(strs,Util.makeTuple,encoding), loadFileThread);
+        System.GC_enable();
+        newp = List.fold(newps, Interactive.updateProgram, p);
+      then
+        (Env.emptyCache(),Values.BOOL(true),GlobalScript.SYMBOLTABLE(newp,aDep,NONE(),ic,iv,cf,lf));
+
+    case (cache,env,"loadFiles",_,st,_)
+      equation
+        System.GC_enable();
+      then (cache,Values.BOOL(false),st);
+
     case (cache,env,"loadString",{Values.STRING(str),Values.STRING(name),Values.STRING(encoding)},
           (st as GlobalScript.SYMBOLTABLE(
             ast = p,depends=aDep,instClsLst = ic,
@@ -3124,7 +3144,7 @@ algorithm
     case (cache,env,"runScriptParallel",{Values.ARRAY(valueLst=vals),Values.INTEGER(i),Values.BOOL(true)},st,_)
       equation
         strs = List.map(vals,ValuesUtil.extractValueString);
-        blst = System.forkCall(i, List.map1(strs, Util.makeTuple, st), Interactive.evaluateFork);
+        blst = System.launchParallelTasks(i, List.map1(strs, Util.makeTuple, st), Interactive.evaluateFork);
         v = ValuesUtil.makeArray(List.map(blst, ValuesUtil.makeBoolean));
       then (cache,v,st);
 
@@ -7304,4 +7324,17 @@ algorithm
   // print(str +& ": " +&  boolString(b) +& "\n");
 end isChanged;
 
+protected function loadFileThread
+  input tuple<String,String> inFileEncoding;
+  output Absyn.Program program;
+algorithm
+  program := matchcontinue inFileEncoding
+    local
+      String filename,encoding;
+    case ((filename,encoding)) then ClassLoader.loadFile(filename,encoding);
+    /* TODO: Add locking mechanism to write to parent's error stream instead of stdout */
+    else equation print(Error.printMessagesStr()); then fail();
+  end matchcontinue;
+end loadFileThread;
+
 end CevalScript;

Compiler/Script/Interactive.mo

@@ -195,8 +195,9 @@ public function evaluateFork
 "As evaluateToStdOut, but takes the inputs as a tuple of mos-script file and symbol table.
 As it is supposed to work in a thread without output, it also flushes the error-buffer since that will otherwise be lost to the void."
   input tuple<String,GlobalScript.SymbolTable> inTpl;
+  output Boolean b;
 algorithm
-  _ := matchcontinue inTpl
+  b := matchcontinue inTpl
     local
       String mosfile;
       GlobalScript.Statements statements;
@@ -210,11 +211,11 @@ algorithm
         statements = Parser.parseexp(mosfile);
         _ = evaluateToStdOut(statements,GlobalScript.SYMBOLTABLE(ast,GlobalScript.emptyDepends,explodedAst,{},{},{},{}),true);
         print(Error.printMessagesStr());
-      then ();
+      then true;
     else
       equation
         print(Error.printMessagesStr());
-      then fail();
+      then false;
   end matchcontinue;
 end evaluateFork;
 

Compiler/Util/System.mo

@@ -1014,17 +1014,19 @@ public function numProcessors
   external "C" result = System_numProcessors() annotation(Library = {"omcruntime"});
 end numProcessors;
 
-public function forkCall "Takes a list of inputs and produces a list of Boolean (true if the function call was successful). The function is called by not using forks (experimental version using threads because fork doesn't play nice)."
+public function launchParallelTasks "Takes a list of inputs and produces a list of Boolean (true if the function call was successful). The function is called by not using forks (experimental version using threads because fork doesn't play nice). Only returns if all functions return."
   input Integer numThreads;
-  input list<Any> inData;
+  input list<AnyInput> inData;
   input ForkFunction func;
-  output list<Boolean> result;
+  output list<AnyOutput> result;
   partial function ForkFunction
-    input Any inData;
+    input AnyInput inData;
+    output AnyOutput outData;
   end ForkFunction;
-  replaceable type Any subtypeof Any;
-external "C" result = System_forkCall(numThreads, inData, func) annotation(Library = {"omcruntime"});
-end forkCall;
+  replaceable type AnyInput subtypeof Any;
+  replaceable type AnyOutput subtypeof Any;
+external "C" result = System_launchParallelTasks(OpenModelica.threadData(), numThreads, inData, func) annotation(Library = {"omcruntime"});
+end launchParallelTasks;
 
 public function exit "Exits the compiler at this point with the given exit status."
   input Integer status;
@@ -1044,4 +1046,12 @@ public function initGarbageCollector "this needs to be called first in Main.mo"
 external "C" System_initGarbageCollector() annotation(Library = {"omcruntime"});
 end initGarbageCollector;
 
+public function GC_enable
+external "C" annotation(Library = {"gc"});
+end GC_enable;
+
+public function GC_disable
+external "C" annotation(Library = {"gc"});
+end GC_disable;
+
 end System;

Compiler/Util/Util.mo

@@ -2535,45 +2535,15 @@ public function getAbsoluteDirectoryAndFile "author: x02lucpo
   splits the filepath in directory and filename
   (\"c:\\programs\\file.mo\") => (\"c:\\programs\",\"file.mo\")
   (\"..\\work\\file.mo\") => (\"c:\\openmodelica123\\work\", \"file.mo\")"
-  input String inString;
-  output String outString1;
-  output String outString2;
+  input String filename;
+  output String dirname;
+  output String basename;
+protected
+  String realpath;
 algorithm
-  (outString1,outString2):=
-  matchcontinue (inString)
-    local
-      String file,pd,path,res,file_1,file_path,dir_path,current_dir,name;
-      list<String> list_path_1,list_path;
-    case (file_1)
-      equation
-        file = replaceSlashWithPathDelimiter(file_1);
-        pd = System.pathDelimiter();
-        /* (pd_chr :: {}) = stringListStringChar(pd); */
-        (path :: {}) = stringSplitAtChar(file, pd) "same dir only filename as param" ;
-        res = System.pwd();
-      then
-        (res,path);
-    case (file_1)
-      equation
-        file = replaceSlashWithPathDelimiter(file_1);
-        pd = System.pathDelimiter();
-        /* (pd_chr :: {}) = stringListStringChar(pd); */
-        list_path = stringSplitAtChar(file, pd);
-        file_path = List.last(list_path);
-        list_path_1 = List.stripLast(list_path);
-        dir_path = stringDelimitList(list_path_1, pd);
-        current_dir = System.pwd();
-        0 = System.cd(dir_path);
-        res = System.pwd();
-        0 = System.cd(current_dir);
-      then
-        (res,file_path);
-    case (name)
-      equation
-        Debug.fprint(Flags.FAILTRACE, "- Util.getAbsoluteDirectoryAndFile failed");
-      then
-        fail();
-  end matchcontinue;
+  realpath := System.realpath(filename);
+  dirname := System.dirname(realpath);
+  basename := System.basename(realpath);
 end getAbsoluteDirectoryAndFile;
 
 

Compiler/runtime/System_omc.c

@@ -302,12 +302,13 @@ extern void* System_strtok(const char *str0, const char *delimit)
   char *s;
   void *res = mmc_mk_nil();
   char *str = GC_strdup(str0);
-  s=strtok(str,delimit);
+  char *saveptr;
+  s=strtok_r(str,delimit,&saveptr);
   if (s == NULL) {
     return res;
   }
   res = mmc_mk_cons(mmc_mk_scon(s),res);
-  while ((s=strtok(NULL,delimit))) {
+  while ((s=strtok_r(NULL,delimit,&saveptr))) {
     res = mmc_mk_cons(mmc_mk_scon(s),res);
   }
   return listReverse(res);
@@ -710,45 +711,64 @@ extern int System_fileIsNewerThan(const char *file1, const char *file2)
   return res;
 }
 
+typedef void* voidp;
+
 /* Work in progress: Threading support in OMC */
 typedef struct thread_data {
   pthread_mutex_t mutex;
-  void (*fn)(void*,void*);
+  void* (*fn)(void*,void*);
   int fail;
   int current;
   int len;
   void **commands;
-  int *status;
+  void **status;
 } thread_data;
 
-static void* System_forkCallThread(void *in)
+static void* System_launchParallelTasksThread(void *in)
 {
   int exitstatus = 1;
   int n;
   thread_data *data = (thread_data*) in;
   while (1) {
+    int fail = 1;
     pthread_mutex_lock(&data->mutex);
     n = data->current++;
     pthread_mutex_unlock(&data->mutex);
     if (data->fail || data->current > data->len) break;
     MMC_TRY_TOP()
     threadData->mmc_thread_work_exit = threadData->mmc_jumper;
-    data->fn(threadData,data->commands[n]);
-    exitstatus = 0;
+    data->status[n] = data->fn(threadData,data->commands[n]);
+    fail = 0;
     MMC_CATCH_TOP()
-    data->status[n] = ! exitstatus;
+    if (fail) {
+      data->fail = 1;
+    }
   }
   return NULL;
 }
 
-extern void* System_forkCall(int numThreads, void *dataLst, void (*fn)(void*))
+static void* System_launchParallelTasksSerial(threadData_t *threadData, void *dataLst, void* (*fn)(void*,void*))
+{
+  void *result = mmc_mk_nil();
+  while (!listEmpty(dataLst)) {
+    result = mmc_mk_cons(fn(threadData, MMC_CAR(dataLst)),result);
+    dataLst = MMC_CDR(dataLst);
+  }
+  return listReverse(dataLst);
+}
+
+extern void* System_launchParallelTasks(threadData_t *threadData, int numThreads, void *dataLst, void* (*fn)(void*,void*))
 {
   int len = listLength(dataLst), i;
   void *commands[len];
-  int status[len], ids[len];
+  void *status[len];
+  int ids[len];
   void *result = mmc_mk_nil();
   pthread_t th[numThreads];
   thread_data data;
+  if (numThreads == 1 || len == 1) {
+    return System_launchParallelTasksSerial(threadData,dataLst,fn);
+  }
   pthread_mutex_init(&data.mutex,NULL);
   data.fn = fn;
   data.current = 0;
@@ -761,13 +781,16 @@ extern void* System_forkCall(int numThreads, void *dataLst, void (*fn)(void*))
   }
   numThreads = min(numThreads,len);
   for (i=0; i<numThreads; i++) {
-    GC_pthread_create(&th[i],NULL,System_forkCallThread,&data);
+    GC_pthread_create(&th[i],NULL,System_launchParallelTasksThread,&data);
   }
   for (i=0; i<numThreads; i++) {
     GC_pthread_join(th[i], NULL);
   }
+  if (data.fail) {
+    MMC_THROW_INTERNAL();
+  }
   for (i=len-1; i>=0; i--) {
-    result = mmc_mk_cons(mmc_mk_icon(status[i]), result);
+    result = mmc_mk_cons(status[i], result);
   }
   return result;
 }
@@ -783,7 +806,7 @@ void System_threadFail(threadData_t *threadData)
     fprintf(stderr, "System_threadFail called in something that is not a worker thread!\nThe application will now exit!\n");
     abort();
   }
-  longjmp(threadData->mmc_thread_work_exit,1);
+  longjmp(*threadData->mmc_thread_work_exit,1);
 }
 
 #ifdef __cplusplus

Compiler/runtime/System_rml.c

@@ -1814,9 +1814,23 @@ RML_BEGIN_LABEL(System__numProcessors)
 }
 RML_END_LABEL
 
-RML_BEGIN_LABEL(System__forkCall)
+RML_BEGIN_LABEL(System__launchParallelTasks)
 {
-  c_add_message(NULL,-1,ErrorType_scripting,ErrorLevel_error,gettext("Fork is not available when OpenModelica is compiled using RML"),NULL,0);
+  c_add_message(NULL,-1,ErrorType_scripting,ErrorLevel_error,gettext("Threads are not available when OpenModelica is compiled using RML"),NULL,0);
+  RML_TAILCALLK(rmlFC);
+}
+RML_END_LABEL
+
+RML_BEGIN_LABEL(System__GC_5fenable)
+{
+  c_add_message(NULL,-1,ErrorType_scripting,ErrorLevel_error,gettext("Boehm GC (GC_enable) is not available when OpenModelica is compiled using RML"),NULL,0);
+  RML_TAILCALLK(rmlFC);
+}
+RML_END_LABEL
+
+RML_BEGIN_LABEL(System__GC_5fdisable)
+{
+  c_add_message(NULL,-1,ErrorType_scripting,ErrorLevel_error,gettext("Boehm GC (GC_disable) is not available when OpenModelica is compiled using RML"),NULL,0);
   RML_TAILCALLK(rmlFC);
 }
 RML_END_LABEL

SimulationRuntime/c/meta/gc/marksweep.c

@@ -588,18 +588,13 @@ void *mmc_alloc_words(unsigned nwords)
 
 #elif defined(_MMC_USE_BOEHM_GC_) /* use the BOEHM Garbage collector */
 
-#if !defined(LARGE_CONFIG)
-#define LARGE_CONFIG 1
-#endif
-#include <gc.h>
-
 static mmc_GC_state_type x_mmc_GC_state;
 mmc_GC_state_type *mmc_GC_state = &x_mmc_GC_state;
 
 /* primary allocation routine for MetaModelica */
 void *mmc_alloc_words(unsigned nwords)
 {
-  return GC_malloc(nwords * sizeof(void*));
+  return GC_MALLOC(nwords * sizeof(void*));
 }
 
 #else /* NO GC */

SimulationRuntime/c/meta/gc/mmc_gc.h

@@ -121,6 +121,7 @@ static inline void mmc_GC_add_roots(modelica_metatype* p, int n, mmc_GC_local_st
 #if defined(_MMC_USE_BOEHM_GC_) /* use the BOEHM Garbage collector */
 
 #define LARGE_CONFIG
+#define GC_REDIRECT_TO_LOCAL
 #include <gc.h>
 #include <pthread.h>
 /* gc.h doesn't include this by default; and the actual header redirects dlopen, which does not have an implementation */
@@ -133,7 +134,7 @@ int GC_pthread_join(pthread_t, void **);
 #define mmc_GC_collect(local_GC_state)
 
 /* Atomic pointers only work correctly if we use untagged pointers */
-#define mmc_alloc_words_atomic(nwords) GC_malloc_atomic((nwords) * sizeof(void*))
+#define mmc_alloc_words_atomic(nwords) GC_MALLOC_ATOMIC((nwords) * sizeof(void*))
 
 #else /* NO_GC */