From 6fd6b26f7144c26bf6eefe314853513ae9afb169 Mon Sep 17 00:00:00 2001 From: Wenduo Zhou Date: Thu, 6 Nov 2014 10:47:05 -0500 Subject: [PATCH 1/6] Refs #10450. Enable unit test to test numerical operation. On the TimeSeriesProperty such as min, max, avarage and etc. On branch feature/10450_record_order_by_runnumber modified: ../Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py --- .../algorithms/ExportExperimentLogTest.py | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/Code/Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py b/Code/Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py index 3b5e3ac84f16..fdcbf1d51177 100644 --- a/Code/Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py +++ b/Code/Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py @@ -20,10 +20,10 @@ def test_exportFileNew(self): # Test algorithm alg_test = run_algorithm("ExportExperimentLog", InputWorkspace = "TestMatrixWS", - OutputFilename = "TestRecord.txt", - SampleLogNames = ["run_number", "duration", "proton_charge"], - SampleLogTitles = ["RUN", "Duration", "ProtonCharge"], - SampleLogOperation = [None, None, "sum"], + OutputFilename = "TestRecord001.txt", + SampleLogNames = ["run_number", "duration", "proton_charge", "proton_charge", "proton_charge"], + SampleLogTitles = ["RUN", "Duration", "ProtonCharge", "MinPCharge", "MeanPCharge"], + SampleLogOperation = [None, None, "sum", "min", "average"], FileMode = "new") # Validate @@ -51,7 +51,22 @@ def test_exportFileNew(self): # Check line firstdataline = lines[1] terms = firstdataline.strip().split("\t") - self.assertEquals(len(terms), 3) + self.assertEquals(len(terms), 5) + + # Get property + pchargelog = ws.getRun().getProperty("proton_charge").value + sumpcharge = numpy.sum(pchargelog) + minpcharge = numpy.min(pchargelog) + avgpcharge = numpy.average(pchargelog) + + v2 = float(terms[2]) + self.assertAlmostEqual(sumpcharge, v2) + v3 = float(terms[3]) + self.assertAlmostEqual(minpcharge, v3) + v4 = float(terms[4]) + self.assertAlmostEqual(avgpcharge, v4) + + # # # Remove generated files From e8de6692dc7f8b3ef5646b646591505e1b36dca0 Mon Sep 17 00:00:00 2001 From: Wenduo Zhou Date: Thu, 6 Nov 2014 14:56:11 -0500 Subject: [PATCH 2/6] Refs #10450. Added sorting functionality to algorithm. And enable the algirthm to remove bad character ('\x0') automatically. Unit test is added for the new features On branch feature/10450_record_order_by_runnumber modified: ../Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py modified: ../Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py --- .../plugins/algorithms/ExportExperimentLog.py | 119 ++++++++++++++++-- .../algorithms/ExportExperimentLogTest.py | 99 ++++++++++++++- 2 files changed, 200 insertions(+), 18 deletions(-) diff --git a/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py b/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py index c5ba13a1e7fc..1cb6ebd168a6 100644 --- a/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py +++ b/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py @@ -42,6 +42,8 @@ def PyInit(self): "With this option, the posfix of the output file is .csv automatically. " self.declareProperty("FileFormat", "tab", mantid.kernel.StringListValidator(fileformates), des) + self.declareProperty("OrderByTitle", "", "Log file will be ordered by the value of this title from low to high.") + # Time zone timezones = ["UTC", "America/New_York"] self.declareProperty("TimeZone", "America/New_York", StringListValidator(timezones)) @@ -81,6 +83,10 @@ def PyExec(self): # Append the new experiment log self._appendExpLog(valuedict) + # Order the record file + if self._orderRecord is True: + self._orderRecordFile() + return def _processInputs(self): @@ -109,13 +115,29 @@ def _processInputs(self): if len(self._headerTitles) > 0 and len(self._headerTitles) != len(self._sampleLogNames): raise NotImplementedError("Input header titles have a different length to sample log names") + # Output file format + self._fileformat = self.getProperty("FileFormat").value + if self._fileformat == "tab": + self._valuesep = "\t" + else: + self._valuesep = "," + + # Output file's postfix + if self._fileformat == "comma (csv)": + fileName, fileExtension = os.path.splitext(self._logfilename) + if fileExtension != ".csv": + # Force the extension of the output file to be .csv + self._logfilename = "%s.csv" % (fileName) + # Determine file mode if os.path.exists(self._logfilename) is False: self._filemode = "new" if len(self._headerTitles) == 0: raise NotImplementedError("Without specifying header title, unable to new a file.") + self.log().debug("[DB] Log file %s does not exist. So file mode is NEW." % (self._logfilename)) else: self._filemode = self.getProperty("FileMode").value + self.log().debug("[DB] FileMode is from user specified value.") # Examine the file mode if self._filemode == "new" or self._filemode == "append": @@ -127,22 +149,20 @@ def _processInputs(self): # This is left for a feature that might be needed in future. self._reorderOld = False - # Output file format - self._fileformat = self.getProperty("FileFormat").value - if self._fileformat == "tab": - self._valuesep = "\t" - else: - self._valuesep = "," - - # Output file's postfix - if self._fileformat == "comma (csv)": - fileName, fileExtension = os.path.splitext(self._logfilename) - if fileExtension != ".csv": - # Force the extension of the output file to be .csv - self._logfilename = "%s.csv" % (fileName) self._timezone = self.getProperty("TimeZone").value + # Determine whether output log-record file should be ordered by value of some log + self._orderRecord = False + self._titleToOrder = None + if self._filemode != "new": + ordertitle = self.getProperty("OrderByTitle").value + if ordertitle in self._headerTitles: + self._orderRecord = True + self.titleToOrder = ordertitle + else: + self.log().warning("Specified title to order by (%s) is not in given log titles." % (ordertitle)) + return def _createLogFile(self): @@ -256,6 +276,78 @@ def _appendExpLog(self, logvaluedict): return + def _orderRecordFile(self): + """ Check and order (if necessary) record file + by value of specified log by title + """ + self.log().notice("[DB] Order Record File!") + + # Read line + lfile = open(self._logfilename, "r") + lines = lfile.readlines() + lfile.close() + + # Create dictionary for the log value + titlelines = [] + linedict = {} + ilog = self._headerTitles.index(self.titleToOrder) + + for line in lines: + cline = line.strip() + if len(cline) == 0: + continue + + if cline.startswith(self._headerTitles[0]) is True or cline.startswith('#'): + # title line or comment line + titlelines.append(line) + else: + # value line + try: + keyvalue = line.split(self._valuesep)[ilog] + except IndexError: + self.log().error("Order record failed.") + return + linedict[keyvalue] = line + # ENDIFELSE + # ENDFOR + + # Check needs to re-order + if linedict.keys() != sorted(linedict.keys()): + # Re-write file + wbuf = "" + + # title line + for line in titlelines: + wbuf += line + + # log entry line + iline = 0 + numlines = len(linedict.keys()) + for ivalue in sorted(linedict.keys()): + wbuf += linedict[ivalue] + # Add extra \n in case reordered + if iline != numlines-1 and wbuf[-1] != '\n': + wbuf += '\n' + iline += 1 + # ENDFOR + + # Last line should not ends with \n + if wbuf[-1] == '\n': + wbuf = wbuf[0:-1] + + # Remove unsupported character which may cause importing error of GNUMERIC + wbuf = wbuf.translate(None, chr(0)) + + # Re-write file + ofile = open(self._logfilename, "w") + ofile.write(wbuf) + ofile.close() + + # ENDIF + + return + + def _reorderExistingFile(self): """ Re-order the columns of the existing experimental log file """ @@ -263,6 +355,7 @@ def _reorderExistingFile(self): return + def _getSampleLogsValue(self): """ From the workspace to get the value """ diff --git a/Code/Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py b/Code/Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py index fdcbf1d51177..3e6f9a592ef1 100644 --- a/Code/Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py +++ b/Code/Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py @@ -66,8 +66,6 @@ def test_exportFileNew(self): v4 = float(terms[4]) self.assertAlmostEqual(avgpcharge, v4) - - # # # Remove generated files os.remove(outfilename) @@ -326,7 +324,94 @@ def test_exportFileNewCSV(self): return - def createTestWorkspace(self): + + def test_sortRecordFile(self): + """ Test to append logs and sort the log record file + """ + # Record 0 + ws1 = self.createTestWorkspace(run=10000) + AnalysisDataService.addOrReplace("TestMatrixWS1", ws1) + + alg_test = run_algorithm("ExportExperimentLog", + InputWorkspace = "TestMatrixWS1", + OutputFilename = "TestRecord9.txt", + SampleLogNames = ["run_number", "duration", "proton_charge"], + SampleLogTitles = ["RUN", "Duration", "ProtonCharge"], + SampleLogOperation = [None, None, "min"], + FileMode = "new", + FileFormat = "comma (csv)", + OrderByTitle = 'RUN') + + + # Record 1 + ws2 = self.createTestWorkspace(run=11000) + AnalysisDataService.addOrReplace("TestMatrixWS2", ws2) + + alg_test = run_algorithm("ExportExperimentLog", + InputWorkspace = "TestMatrixWS2", + OutputFilename = "TestRecord9.txt", + SampleLogNames = ["run_number", "duration", "proton_charge"], + SampleLogTitles = ["RUN", "Duration", "ProtonCharge"], + SampleLogOperation = [None, None, "min"], + FileMode = "fastappend", + FileFormat = "comma (csv)", + OrderByTitle = 'RUN') + + # Record 2 + ws3 = self.createTestWorkspace(run=10023) + AnalysisDataService.addOrReplace("TestMatrixWS3", ws3) + + alg_test = run_algorithm("ExportExperimentLog", + InputWorkspace = "TestMatrixWS3", + OutputFilename = "TestRecord9.txt", + SampleLogNames = ["run_number", "duration", "proton_charge"], + SampleLogTitles = ["RUN", "Duration", "ProtonCharge"], + SampleLogOperation = [None, None, "min"], + FileMode = "fastappend", + FileFormat = "comma (csv)", + OrderByTitle = 'RUN') + + # Verify + # Locate file + outfilename = alg_test.getProperty("OutputFilename").value.split(".txt")[0] + ".csv" + try: + ifile = open(outfilename) + lines = ifile.readlines() + ifile.close() + except IOError as err: + print "Unable to open file %s. " % (outfilename) + self.assertTrue(False) + return + + # Last line cannot be empty, i.e., before EOF '\n' is not allowed + lastline = lines[-1] + self.assertTrue(len(lastline.strip()) > 0) + + # Number of lines + self.assertEquals(len(lines), 4) + + # Check value + for i in xrange(1, 3): + currline = lines[i] + curr_run = int(currline.split(",")[0]) + curr_min = float(currline.split(",")[2]) + nextline = lines[i+1] + next_run = int(nextline.split(',')[0]) + next_min = float(nextline.split(',')[2]) + self.assertTrue(curr_run < next_run) + self.assertTrue(curr_min < next_min) + + + # Remove generated files + os.remove(outfilename) + AnalysisDataService.remove("TestMatrixWS1") + AnalysisDataService.remove("TestMatrixWS2") + AnalysisDataService.remove("TestMatrixWS3") + + return + + + def createTestWorkspace(self, run=23456): """ Create a workspace for testing against with ideal log values """ from mantid.simpleapi import CreateWorkspace @@ -348,10 +433,14 @@ def createTestWorkspace(self): tsp_b=kernel.FloatTimeSeriesProperty("SensorA") for i in arange(25): tmptime = strftime("%Y-%m-%d %H:%M:%S", gmtime(mktime(gmtime())+i)) - tsp_a.addValue(tmptime, 1.0*i*i) + if run == 23456: + shift = 0 + else: + shift = int(run) + tsp_a.addValue(tmptime, 1.0*i*i + shift) tsp_b.addValue(tmptime, 1.234*(i+1)) - wksp.mutableRun()['run_number']="23456" + wksp.mutableRun()['run_number']=str(run) wksp.mutableRun()['duration']=342.3 wksp.mutableRun()['SensorA'] = tsp_b wksp.mutableRun()['proton_charge']=tsp_a From aaa91ef7b1c98bd435783e6997ce5345fa0324d6 Mon Sep 17 00:00:00 2001 From: Wenduo Zhou Date: Fri, 7 Nov 2014 15:00:26 -0500 Subject: [PATCH 3/6] Refs #10450. Added a new property to algorithm. to override the log value from workspace. and a new unit test is added for the new feature. On branch feature/10450_record_order_by_runnumber modified: ../Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py modified: ../Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py --- .../plugins/algorithms/ExportExperimentLog.py | 56 +++++++++-- .../algorithms/ExportExperimentLogTest.py | 95 +++++++++++++++++++ 2 files changed, 142 insertions(+), 9 deletions(-) diff --git a/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py b/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py index 1cb6ebd168a6..7c7434ef3106 100644 --- a/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py +++ b/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py @@ -44,6 +44,9 @@ def PyInit(self): self.declareProperty("OrderByTitle", "", "Log file will be ordered by the value of this title from low to high.") + overrideprop = StringArrayProperty("OverrideLogValue", values=[], direction=Direction.Input) + self.declareProperty(overrideprop, "List of paired strings as log title and value to override values from workspace.") + # Time zone timezones = ["UTC", "America/New_York"] self.declareProperty("TimeZone", "America/New_York", StringListValidator(timezones)) @@ -149,7 +152,6 @@ def _processInputs(self): # This is left for a feature that might be needed in future. self._reorderOld = False - self._timezone = self.getProperty("TimeZone").value # Determine whether output log-record file should be ordered by value of some log @@ -163,6 +165,19 @@ def _processInputs(self): else: self.log().warning("Specified title to order by (%s) is not in given log titles." % (ordertitle)) + # Override log values: it will not work in fastappend mode to override + overridelist = self.getProperty("OverrideLogValue").value + if len(self._headerTitles) > 0: + if len(overridelist) % 2 != 0: + raise NotImplementedError("Number of items in OverrideLogValue must be even.") + self._ovrdTitleValueDict = {} + for i in xrange(len(overridelist)/2): + title = overridelist[2*i] + if title in self._headerTitles: + self._ovrdTitleValueDict[title] = overridelist[2*i+1] + else: + self.log().warning("Override title %s is not recognized. " % (title)) + return def _createLogFile(self): @@ -210,10 +225,14 @@ def _examineLogFile(self): # Examine titles = titleline.split() + self.log().notice("[DB] Examine finds titles: %s" % (titles)) same = True if len(titles) != len(self._headerTitles): - same = False + if len(self._headerTitles) == 0: + self._headerTitles = titles[:] + else: + same = False for ititle in xrange(len(titles)): title1 = titles[ititle] title2 = self._headerTitles[ititle] @@ -258,14 +277,33 @@ def _appendExpLog(self, logvaluedict): # Write to a buffer wbuf = "" + self.log().notice("[DB] Samlpe Log Names: %s" % (self._sampleLogNames)) + self.log().notice("[DB] Title Names: %s" % (self._headerTitles)) + + if len(self._headerTitles) == 0: + skip = True + else: + skip = False + + headertitle = None for il in xrange(len(self._sampleLogNames)): - logname = self._sampleLogNames[il] - optype = self._sampleLogOperations[il] - key = logname + "-" + optype - if key in logvaluedict.keys(): - value = logvaluedict[key] - elif logname in logvaluedict.keys(): - value = logvaluedict[logname] + if skip is False: + headertitle = self._headerTitles[il] + if headertitle is not None and headertitle in self._ovrdTitleValueDict.keys(): + # overriden + value = self._ovrdTitleValueDict[headertitle] + + else: + # from input workspace + logname = self._sampleLogNames[il] + optype = self._sampleLogOperations[il] + key = logname + "-" + optype + if key in logvaluedict.keys(): + value = logvaluedict[key] + elif logname in logvaluedict.keys(): + value = logvaluedict[logname] + # ENDIFELSE + wbuf += "%s%s" % (str(value), self._valuesep) wbuf = wbuf[0:-1] diff --git a/Code/Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py b/Code/Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py index 3e6f9a592ef1..3f2e1d66e514 100644 --- a/Code/Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py +++ b/Code/Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py @@ -410,6 +410,101 @@ def test_sortRecordFile(self): return + def test_sortRecordFileOverride(self): + """ Test to append logs and sort the log record file + """ + # Record 0 + ws1 = self.createTestWorkspace(run=10000) + AnalysisDataService.addOrReplace("TestMatrixWS1", ws1) + + alg_test = run_algorithm("ExportExperimentLog", + InputWorkspace = "TestMatrixWS1", + OutputFilename = "TestRecord10.txt", + SampleLogNames = ["run_number", "duration", "proton_charge", "proton_charge"], + SampleLogTitles = ["RUN", "Duration", "ProtonCharge", "ProtonCharge-Avg"], + SampleLogOperation = [None, None, "min", "average"], + FileMode = "new", + FileFormat = "tab", + OverrideLogValue = ["Duration", "12345", "ProtonCharge-Avg", "32.921"], + OrderByTitle = 'RUN') + + + # Record 1 + ws2 = self.createTestWorkspace(run=11000) + AnalysisDataService.addOrReplace("TestMatrixWS2", ws2) + + alg_test = run_algorithm("ExportExperimentLog", + InputWorkspace = "TestMatrixWS2", + OutputFilename = "TestRecord10.txt", + SampleLogNames = ["run_number", "duration", "proton_charge", "proton_charge"], + SampleLogTitles = ["RUN", "Duration", "ProtonCharge", "ProtonCharge-Avg"], + SampleLogOperation = [None, None, "min", "average"], + FileMode = "fastappend", + FileFormat = "tab", + OverrideLogValue = ["Duration", "23456", "ProtonCharge-Avg", "22.921"], + OrderByTitle = 'RUN') + + # Record 2 + ws3 = self.createTestWorkspace(run=10023) + AnalysisDataService.addOrReplace("TestMatrixWS3", ws3) + + alg_test = run_algorithm("ExportExperimentLog", + InputWorkspace = "TestMatrixWS3", + OutputFilename = "TestRecord10.txt", + SampleLogNames = ["run_number", "duration", "proton_charge", "proton_charge"], + SampleLogTitles = ["RUN", "Duration", "ProtonCharge", "ProtonCharge-Avg"], + SampleLogOperation = [None, None, "min", "average"], + FileMode = "fastappend", + FileFormat = "tab", + OverrideLogValue = ["Duration", "34567", "ProtonCharge-Avg", "12.921"], + OrderByTitle = 'RUN') + + # Verify + # Locate file + outfilename = alg_test.getProperty("OutputFilename").value + try: + ifile = open(outfilename) + lines = ifile.readlines() + ifile.close() + except IOError as err: + print "Unable to open file %s. " % (outfilename) + self.assertTrue(False) + return + + # Last line cannot be empty, i.e., before EOF '\n' is not allowed + lastline = lines[-1] + self.assertTrue(len(lastline.strip()) > 0) + + # Number of lines + self.assertEquals(len(lines), 4) + + # Check value + for i in xrange(1, 3): + currline = lines[i] + curr_run = int(currline.split("\t")[0]) + curr_min = float(currline.split("\t")[2]) + nextline = lines[i+1] + next_run = int(nextline.split('\t')[0]) + next_min = float(nextline.split('\t')[2]) + self.assertTrue(curr_run < next_run) + self.assertTrue(curr_min < next_min) + + line2 = lines[2] + terms = line2.split("\t") + duration = int(terms[1]) + self.assertEquals(duration, 34567) + pchargeavg = float(terms[3]) + self.assertAlmostEqual(pchargeavg, 12.921) + + + # Remove generated files + os.remove(outfilename) + AnalysisDataService.remove("TestMatrixWS1") + AnalysisDataService.remove("TestMatrixWS2") + AnalysisDataService.remove("TestMatrixWS3") + + return + def createTestWorkspace(self, run=23456): """ Create a workspace for testing against with ideal log values From 686c162db2dab83ee1b9856431fb25166d4b0546 Mon Sep 17 00:00:00 2001 From: Wenduo Zhou Date: Mon, 10 Nov 2014 10:47:37 -0500 Subject: [PATCH 4/6] Refs #10450. Consider the case of duplicated records. --- .../plugins/algorithms/ExportExperimentLog.py | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py b/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py index 7c7434ef3106..9a95b8d7f95c 100644 --- a/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py +++ b/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py @@ -43,10 +43,12 @@ def PyInit(self): self.declareProperty("FileFormat", "tab", mantid.kernel.StringListValidator(fileformates), des) self.declareProperty("OrderByTitle", "", "Log file will be ordered by the value of this title from low to high.") + self.declareProperty("RemoveDuplicateRecord", False, "Coupled with OrderByTitle, duplicated record will be removed.") overrideprop = StringArrayProperty("OverrideLogValue", values=[], direction=Direction.Input) self.declareProperty(overrideprop, "List of paired strings as log title and value to override values from workspace.") + # Time zone timezones = ["UTC", "America/New_York"] self.declareProperty("TimeZone", "America/New_York", StringListValidator(timezones)) @@ -161,10 +163,14 @@ def _processInputs(self): ordertitle = self.getProperty("OrderByTitle").value if ordertitle in self._headerTitles: self._orderRecord = True + self._removeDupRecord = self.getProperty("RemoveDuplicateRecord").value self.titleToOrder = ordertitle else: self.log().warning("Specified title to order by (%s) is not in given log titles." % (ordertitle)) + if self._orderRecord is False: + self._removeDupRecord = False + # Override log values: it will not work in fastappend mode to override overridelist = self.getProperty("OverrideLogValue").value if len(self._headerTitles) > 0: @@ -330,6 +336,7 @@ def _orderRecordFile(self): linedict = {} ilog = self._headerTitles.index(self.titleToOrder) + totnumlines = 0 for line in lines: cline = line.strip() if len(cline) == 0: @@ -341,11 +348,14 @@ def _orderRecordFile(self): else: # value line try: - keyvalue = line.split(self._valuesep)[ilog] + keyvalue = line.split(self._valuesep)[ilog].strip() except IndexError: self.log().error("Order record failed.") return - linedict[keyvalue] = line + if linedict.has_key(keyvalue) is False: + linedict[keyvalue] = [] + linedict[keyvalue].append(line) + totnumlines += 1 # ENDIFELSE # ENDFOR @@ -359,14 +369,15 @@ def _orderRecordFile(self): wbuf += line # log entry line - iline = 0 - numlines = len(linedict.keys()) + numlines = 0 for ivalue in sorted(linedict.keys()): - wbuf += linedict[ivalue] - # Add extra \n in case reordered - if iline != numlines-1 and wbuf[-1] != '\n': - wbuf += '\n' - iline += 1 + for line in linedict[ivalue]: + wbuf += line + # Add extra \n in case reordered + if numlines != totnumlines-1 and wbuf[-1] != '\n': + wbuf += '\n' + numlines += 1 + # ENDFOR # ENDFOR # Last line should not ends with \n From 995991d65b720597c155bec035956b69b334d84b Mon Sep 17 00:00:00 2001 From: Wenduo Zhou Date: Mon, 10 Nov 2014 11:49:54 -0500 Subject: [PATCH 5/6] Refs #10450. Added an optional feature to remove records. Optional feature to have remove records with duplicated value in sample log that is to be ordered by. --- .../plugins/algorithms/ExportExperimentLog.py | 20 ++- .../algorithms/ExportExperimentLogTest.py | 124 ++++++++++++++++++ 2 files changed, 142 insertions(+), 2 deletions(-) diff --git a/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py b/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py index 9a95b8d7f95c..a94f5b0bd038 100644 --- a/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py +++ b/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py @@ -370,13 +370,29 @@ def _orderRecordFile(self): # log entry line numlines = 0 + + # consider the mode to remove duplicate + if self._removeDupRecord is True: + totnumlines = len(linedict.keys()) + for ivalue in sorted(linedict.keys()): - for line in linedict[ivalue]: + if self._removeDupRecord is True: + # If duplicated records is to be removed, only consider the last record + line = linedict[ivalue][-1] wbuf += line - # Add extra \n in case reordered if numlines != totnumlines-1 and wbuf[-1] != '\n': wbuf += '\n' numlines += 1 + + else: + # Consider all! + for line in linedict[ivalue]: + wbuf += line + # Add extra \n in case reordered + if numlines != totnumlines-1 and wbuf[-1] != '\n': + wbuf += '\n' + numlines += 1 + # ENDFOR # ENDFOR # ENDFOR diff --git a/Code/Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py b/Code/Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py index 3f2e1d66e514..eaccc11df168 100644 --- a/Code/Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py +++ b/Code/Mantid/Framework/PythonInterface/test/python/plugins/algorithms/ExportExperimentLogTest.py @@ -410,6 +410,130 @@ def test_sortRecordFile(self): return + def test_removeDupRecord(self): + """ Test to append logs and sort the log record file + """ + # Record 0 + ws1 = self.createTestWorkspace(run=10000) + AnalysisDataService.addOrReplace("TestMatrixWS1", ws1) + + alg_test = run_algorithm("ExportExperimentLog", + InputWorkspace = "TestMatrixWS1", + OutputFilename = "TestRecord11.txt", + SampleLogNames = ["run_number", "duration", "proton_charge", "proton_charge"], + SampleLogTitles = ["RUN", "Duration", "ProtonCharge", "ProtonCharge-Avg"], + SampleLogOperation = [None, None, "min", "average"], + FileMode = "new", + FileFormat = "tab", + OverrideLogValue = ["Duration", "12345", "ProtonCharge-Avg", "32.921"], + RemoveDuplicateRecord = True, + OrderByTitle = 'RUN') + + # Record 0B + alg_test = run_algorithm("ExportExperimentLog", + InputWorkspace = "TestMatrixWS1", + OutputFilename = "TestRecord11.txt", + SampleLogNames = ["run_number", "duration", "proton_charge", "proton_charge"], + SampleLogTitles = ["RUN", "Duration", "ProtonCharge", "ProtonCharge-Avg"], + SampleLogOperation = [None, None, "min", "average"], + FileMode = "fastappend", + FileFormat = "tab", + OverrideLogValue = ["Duration", "12345", "ProtonCharge-Avg", "32.921"], + RemoveDuplicateRecord = True, + OrderByTitle = 'RUN') + + # Record 1 + ws2 = self.createTestWorkspace(run=11000) + AnalysisDataService.addOrReplace("TestMatrixWS2", ws2) + + alg_test = run_algorithm("ExportExperimentLog", + InputWorkspace = "TestMatrixWS2", + OutputFilename = "TestRecord11.txt", + SampleLogNames = ["run_number", "duration", "proton_charge", "proton_charge"], + SampleLogTitles = ["RUN", "Duration", "ProtonCharge", "ProtonCharge-Avg"], + SampleLogOperation = [None, None, "min", "average"], + FileMode = "fastappend", + FileFormat = "tab", + OverrideLogValue = ["Duration", "23456", "ProtonCharge-Avg", "22.921"], + RemoveDuplicateRecord = True, + OrderByTitle = 'RUN') + + # Record 2 + ws3 = self.createTestWorkspace(run=10023) + AnalysisDataService.addOrReplace("TestMatrixWS3", ws3) + + alg_test = run_algorithm("ExportExperimentLog", + InputWorkspace = "TestMatrixWS3", + OutputFilename = "TestRecord11.txt", + SampleLogNames = ["run_number", "duration", "proton_charge", "proton_charge"], + SampleLogTitles = ["RUN", "Duration", "ProtonCharge", "ProtonCharge-Avg"], + SampleLogOperation = [None, None, "min", "average"], + FileMode = "fastappend", + FileFormat = "tab", + OverrideLogValue = ["Duration", "34567", "ProtonCharge-Avg", "12.921"], + RemoveDuplicateRecord = True, + OrderByTitle = 'RUN') + + # Record 2B + alg_test = run_algorithm("ExportExperimentLog", + InputWorkspace = "TestMatrixWS3", + OutputFilename = "TestRecord11.txt", + SampleLogNames = ["run_number", "duration", "proton_charge", "proton_charge"], + SampleLogTitles = ["RUN", "Duration", "ProtonCharge", "ProtonCharge-Avg"], + SampleLogOperation = [None, None, "min", "average"], + FileMode = "fastappend", + FileFormat = "tab", + OverrideLogValue = ["Duration", "34567", "ProtonCharge-Avg", "12.921"], + RemoveDuplicateRecord = True, + OrderByTitle = 'RUN') + + # Verify + # Locate file + outfilename = alg_test.getProperty("OutputFilename").value + try: + ifile = open(outfilename) + lines = ifile.readlines() + ifile.close() + except IOError as err: + print "Unable to open file %s. " % (outfilename) + self.assertTrue(False) + return + + # Last line cannot be empty, i.e., before EOF '\n' is not allowed + lastline = lines[-1] + self.assertTrue(len(lastline.strip()) > 0) + + # Number of lines + self.assertEquals(len(lines), 4) + + # Check value + for i in xrange(1, 3): + currline = lines[i] + curr_run = int(currline.split("\t")[0]) + curr_min = float(currline.split("\t")[2]) + nextline = lines[i+1] + next_run = int(nextline.split('\t')[0]) + next_min = float(nextline.split('\t')[2]) + self.assertTrue(curr_run < next_run) + self.assertTrue(curr_min < next_min) + + line2 = lines[2] + terms = line2.split("\t") + duration = int(terms[1]) + self.assertEquals(duration, 34567) + pchargeavg = float(terms[3]) + self.assertAlmostEqual(pchargeavg, 12.921) + + + # Remove generated files + os.remove(outfilename) + AnalysisDataService.remove("TestMatrixWS1") + AnalysisDataService.remove("TestMatrixWS2") + AnalysisDataService.remove("TestMatrixWS3") + + return + + def test_sortRecordFileOverride(self): """ Test to append logs and sort the log record file """ From ae7ede06e9e5dfe58342e0b8aab4d9de2b92c671 Mon Sep 17 00:00:00 2001 From: Wenduo Zhou Date: Mon, 24 Nov 2014 12:01:06 -0500 Subject: [PATCH 6/6] Refs #10450. Cleaned up the log output. --- .../plugins/algorithms/ExportExperimentLog.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py b/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py index a94f5b0bd038..dece17ba188d 100644 --- a/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py +++ b/Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py @@ -139,17 +139,17 @@ def _processInputs(self): self._filemode = "new" if len(self._headerTitles) == 0: raise NotImplementedError("Without specifying header title, unable to new a file.") - self.log().debug("[DB] Log file %s does not exist. So file mode is NEW." % (self._logfilename)) + self.log().debug("Log file %s does not exist. So file mode is NEW." % (self._logfilename)) else: self._filemode = self.getProperty("FileMode").value - self.log().debug("[DB] FileMode is from user specified value.") + self.log().debug("FileMode is from user specified value.") # Examine the file mode if self._filemode == "new" or self._filemode == "append": if len(self._headerTitles) != len(self._sampleLogNames): raise NotImplementedError("In mode new or append, there must be same number of sample titles and names") - self.log().notice("File mode is %s. " % (self._filemode)) + self.log().information("File mode is %s. " % (self._filemode)) # This is left for a feature that might be needed in future. self._reorderOld = False @@ -231,7 +231,7 @@ def _examineLogFile(self): # Examine titles = titleline.split() - self.log().notice("[DB] Examine finds titles: %s" % (titles)) + self.log().debug("Examine finds titles: %s" % (titles)) same = True if len(titles) != len(self._headerTitles): @@ -283,8 +283,8 @@ def _appendExpLog(self, logvaluedict): # Write to a buffer wbuf = "" - self.log().notice("[DB] Samlpe Log Names: %s" % (self._sampleLogNames)) - self.log().notice("[DB] Title Names: %s" % (self._headerTitles)) + self.log().debug("Samlpe Log Names: %s" % (self._sampleLogNames)) + self.log().debug("Title Names: %s" % (self._headerTitles)) if len(self._headerTitles) == 0: skip = True @@ -324,7 +324,7 @@ def _orderRecordFile(self): """ Check and order (if necessary) record file by value of specified log by title """ - self.log().notice("[DB] Order Record File!") + self.log().debug("Order Record File!") # Read line lfile = open(self._logfilename, "r")