be more careful when opening files for reading; and if there is an er…

…ror, propagate it all the way up to web interface

source/g.py

@@ -204,8 +204,11 @@ def log(loglevel, msg):
         msg += "\n"
         sys.stderr.write(msg)
         if logFileName is not None:
-            with open(logFileName, "a+") as f:
-                f.write(msg)
+            try:
+                with open(logFileName, "a+") as f:
+                    f.write(msg)
+            except Exception as ex:
+                print("Got an exception while trying to write to log file: {}".format(ex))
 
 ################################################
 # Startup
@@ -247,10 +250,13 @@ def prepare(configFileName):
         logFileName = os.path.join(workDir, "spacescanner.log")
 
     # update the log file
-    with open(logFileName, "a+") as f:
-        f.write("============= ")
-        f.write(spacescannerStartTime)
-        f.write("\n")
+    try:
+        with open(logFileName, "a+") as f:
+            f.write("============= ")
+            f.write(spacescannerStartTime)
+            f.write("\n")
+    except Exception as ex:
+        print("Got an exception while preparing log file: {}".format(ex))
 
     try:
         # copy the used configuration file as a reference

source/job.py

@@ -67,6 +67,7 @@ def __init__(self, pool, params, maxCores, areParametersChangeable):
         self.bestOfValueAfter3Sec = None
         # wait this much seconds for other runners to terminate after the first one
         self.waitForTerminationSec = 10
+        self.errorMsg = ""
 
 
     def getFullName(self):
@@ -117,7 +118,8 @@ def createRunners(self):
         for id in range(int(g.getConfig("optimization.runsPerJob"))):
             r = runner.Runner(self, id + 1, self.currentMethod, self.runnerGeneration)
             if not r.prepare(self.workDir, self.copasiFile, bestParams):
-                g.log(LOG_ERROR, "{}: failed to create a runner".format(r.getName()))
+                g.log(LOG_ERROR, "{}: failed to create a runner: {}".format(r.getName(), r.errorMsg))
+                self.errorMsg = r.errorMsg
                 return False
             self.runners.append(r)
 

source/jobpool.py

@@ -44,6 +44,7 @@ def __init__(self, strategy, parameterSets, areParametersChangeable):
         self.maxNumParallelJobs = int(math.ceil(float(self.numUsableCores) / self.numRunnersPerJob))
         self.bestOfValue = strategy.getInitialOfValue()
         self.bestParams = []
+        self.errorMsg = ""
 
     def start(self):
         # try to start n jobs at once
@@ -82,7 +83,11 @@ def nextJob(self):
 
         # execute the job
         if not j.execute(g.workDir, self.strategy.copasiFile):
-            g.log(LOG_DEBUG, "failed to execute {}".format(j.getName()))
+            g.log(LOG_ERROR, "failed to execute {}: {}".format(j.getName(), j.errorMsg))
+            if self.errorMsg == "":
+                self.errorMsg = j.errorMsg
+            elif j.errorMsg != "":
+                self.errorMsg += "\n" + j.errorMsg
             self.finishJob(j)
 
             # while the number is not under the limit, wait for some job to terminate

source/runner.py

@@ -95,6 +95,7 @@ def __init__(self, job, id, methodName, generation):
         self.reportFilename = None
         self.copasiFile = None
         self.generation = generation
+        self.errorMsg = ""
 
     def getFullName(self):
         return self.job.getFullName() + "/{} (method '{}')".format(
@@ -105,6 +106,7 @@ def getName(self):
             self.id, self.methodName)
 
     def prepare(self, workDir, copasiFile, startParamValues):
+        self.errorMsg = ""
         filename = "job{}_runner{}".format(self.job.id, self.id)
 
         # Use separate directory for each run to avoid too many files per directory
@@ -119,9 +121,17 @@ def prepare(self, workDir, copasiFile, startParamValues):
         self.inputFilename = os.path.join(dirname, "input_" + filename + ".cps")
         self.reportFilename = os.path.join(dirname, "output_" + filename + ".log")
         self.copasiFile = copasiFile
-        if not copasiFile.createCopy(self.inputFilename, self.reportFilename,
-                                     self.job.params, [self.methodName], startParamValues,
-                                     self.job.areParametersChangeable):
+
+        try:
+            ok = copasiFile.createCopy(self.inputFilename, self.reportFilename,
+                                       self.job.params, [self.methodName], startParamValues,
+                                       self.job.areParametersChangeable)
+        except Exception as ex:
+            self.errorMsg = "cannot copy the model file {}: {}".format(self.inputFilename, ex)
+            g.log(LOG_ERROR, self.errorMsg)
+            ok = False
+
+        if not ok:
             return False
 
         # rename the old report file, if any expected
@@ -139,7 +149,8 @@ def prepare(self, workDir, copasiFile, startParamValues):
 
         copasiExe = os.path.join(COPASI_DIR, COPASI_EXECUTABLE)
         if not isExecutable(copasiExe):
-            g.log(LOG_ERROR, 'COPASI binary is not executable or does not exist under "' + copasiExe + '"')
+            self.errorMsg = 'COPASI binary is not executable or does not exist under "' + copasiExe + '"'
+            g.log(LOG_ERROR, self.errorMsg)
             return False
 
         args = [copasiExe, "--nologo", self.inputFilename]
@@ -221,9 +232,14 @@ def cleanup(self):
         if stats.isValid:
             # overwrite the input file with parameter values corresponding to the best result so far
             paramsDict = dict(zip(self.job.params, stats.params))
-            self.copasiFile.createCopy(self.inputFilename, self.reportFilename,
-                                       self.job.params, [self.methodName],
-                                       paramsDict, self.job.areParametersChangeable)
+            try:
+                self.copasiFile.createCopy(self.inputFilename, self.reportFilename,
+                                           self.job.params, [self.methodName],
+                                           paramsDict, self.job.areParametersChangeable)
+            except Exception as ex:
+                self.errorMsg = 'Cannot create the final model file {} with the best results: {}'.format(self.inputFilename, ex)
+                g.log(LOG_ERROR, self.errorMsg)
+
 
     def checkReport(self, hasTerminated, now):
         assert self.isActive

source/strategy.py

@@ -400,13 +400,16 @@ def dumpResults(self, totalLimit = 0):
         allParams = self.copasiConfig["params"]
 
         rank = 1
-        with open(filename, "w") as f:
-            self.dumpCsvFileHeader(f)
-            for job in allJobsByBestOfValue:
-                job.dumpResults(f, rank, allParams)
-                if totalLimit and rank >= totalLimit:
-                    break
-                rank += 1
+        try:
+            with open(filename, "w") as f:
+                self.dumpCsvFileHeader(f)
+                for job in allJobsByBestOfValue:
+                    job.dumpResults(f, rank, allParams)
+                    if totalLimit and rank >= totalLimit:
+                        break
+                    rank += 1
+        except Exception as ex:
+            g.log(LOG_ERROR, "cannot save results to {}: {}".format(filename, ex))
 
         g.log(LOG_INFO, '<spacescanner>: results of finished jobs saved in "' + filename + '"')
         return filename
@@ -705,6 +708,15 @@ def finishActivePool(self):
         self.dumpResults()
 
 
+    def updateErrorMsg(self, pool):
+        if pool.errorMsg:
+            if self.lastError == "":
+                self.lastError = pool.errorMsg
+            elif pool.errorMsg != "":
+                self.lastError += "\n" + pool.errorMsg
+                pool.errorMsg = ""
+
+
     def execute(self):
         parameterSelections = []
 
@@ -773,11 +785,15 @@ def execute(self):
                     self.activeJobPool = pool
 
                 pool.start()
+                self.updateErrorMsg(pool)
+
                 while True:
                     time.sleep(1.0)
                     if self.doQuitFlag:
                         return True
 
+                    self.updateErrorMsg(pool)
+
                     try:
                         pool.refresh()
                     except Exception as e:

source/webserver.py

@@ -111,8 +111,11 @@ def saveInFile(self, contents, filename):
         except:
             pass
         filename = os.path.join(dirname, filename)
-        with open(filename, "wb") as f:
-            f.write(contents.encode("UTF-8"))
+        try:
+            with open(filename, "wb") as f:
+                f.write(contents.encode("UTF-8"))
+        except:
+            g.log(LOG_ERROR, "cannot save to file")
         return filename
 
     def sendDefaultHeaders(self, contents, isJSON = True, contentType = None):
@@ -209,7 +212,7 @@ def do_GET(self):
                         "resultsPresent" : sm.getNumFinishedJobs() > 0,
                         "error" : sm.lastError }
             # XXX: perhaps resetting this should be timer based?
-            sm.lastError = None
+            sm.lastError = ""
         elif o.path == '/allstatus':
             response = sm.ioGetAllJobs(qs)
         elif o.path == '/activestatus':