Skip to content

Commit

Permalink
Merge branch 'feature/10450_record_order_by_runnumber'
Browse files Browse the repository at this point in the history
  • Loading branch information
@martyngigg
martyngigg committed Dec 8, 2014
2 parents 2e287f5 + ae7ede0 commit 391f852
Show file tree
Hide file tree
Showing 2 changed files with 512 additions and 31 deletions.
204 changes: 181 additions & 23 deletions Code/Mantid/Framework/PythonInterface/plugins/algorithms/ExportExperimentLog.py
View file
Expand Up @@ -42,6 +42,13 @@ def PyInit(self):
"With this option, the posfix of the output file is .csv automatically. "
self.declareProperty("FileFormat", "tab", mantid.kernel.StringListValidator(fileformates), des)

self.declareProperty("OrderByTitle", "", "Log file will be ordered by the value of this title from low to high.")
self.declareProperty("RemoveDuplicateRecord", False, "Coupled with OrderByTitle, duplicated record will be removed.")

overrideprop = StringArrayProperty("OverrideLogValue", values=[], direction=Direction.Input)
self.declareProperty(overrideprop, "List of paired strings as log title and value to override values from workspace.")


# Time zone
timezones = ["UTC", "America/New_York"]
self.declareProperty("TimeZone", "America/New_York", StringListValidator(timezones))
Expand Down Expand Up @@ -81,6 +88,10 @@ def PyExec(self):
# Append the new experiment log
self._appendExpLog(valuedict)

# Order the record file
if self._orderRecord is True:
self._orderRecordFile()

return

def _processInputs(self):
Expand Down Expand Up @@ -109,40 +120,70 @@ def _processInputs(self):
if len(self._headerTitles) > 0 and len(self._headerTitles) != len(self._sampleLogNames):
raise NotImplementedError("Input header titles have a different length to sample log names")

# Output file format
self._fileformat = self.getProperty("FileFormat").value
if self._fileformat == "tab":
self._valuesep = "\t"
else:
self._valuesep = ","

# Output file's postfix
if self._fileformat == "comma (csv)":
fileName, fileExtension = os.path.splitext(self._logfilename)
if fileExtension != ".csv":
# Force the extension of the output file to be .csv
self._logfilename = "%s.csv" % (fileName)

# Determine file mode
if os.path.exists(self._logfilename) is False:
self._filemode = "new"
if len(self._headerTitles) == 0:
raise NotImplementedError("Without specifying header title, unable to new a file.")
self.log().debug("Log file %s does not exist. So file mode is NEW." % (self._logfilename))
else:
self._filemode = self.getProperty("FileMode").value
self.log().debug("FileMode is from user specified value.")

# Examine the file mode
if self._filemode == "new" or self._filemode == "append":
if len(self._headerTitles) != len(self._sampleLogNames):
raise NotImplementedError("In mode new or append, there must be same number of sample titles and names")

self.log().notice("File mode is %s. " % (self._filemode))
self.log().information("File mode is %s. " % (self._filemode))

# This is left for a feature that might be needed in future.
self._reorderOld = False

# Output file format
self._fileformat = self.getProperty("FileFormat").value
if self._fileformat == "tab":
self._valuesep = "\t"
else:
self._valuesep = ","

# Output file's postfix
if self._fileformat == "comma (csv)":
fileName, fileExtension = os.path.splitext(self._logfilename)
if fileExtension != ".csv":
# Force the extension of the output file to be .csv
self._logfilename = "%s.csv" % (fileName)

self._timezone = self.getProperty("TimeZone").value

# Determine whether output log-record file should be ordered by value of some log
self._orderRecord = False
self._titleToOrder = None
if self._filemode != "new":
ordertitle = self.getProperty("OrderByTitle").value
if ordertitle in self._headerTitles:
self._orderRecord = True
self._removeDupRecord = self.getProperty("RemoveDuplicateRecord").value
self.titleToOrder = ordertitle
else:
self.log().warning("Specified title to order by (%s) is not in given log titles." % (ordertitle))

if self._orderRecord is False:
self._removeDupRecord = False

# Override log values: it will not work in fastappend mode to override
overridelist = self.getProperty("OverrideLogValue").value
if len(self._headerTitles) > 0:
if len(overridelist) % 2 != 0:
raise NotImplementedError("Number of items in OverrideLogValue must be even.")
self._ovrdTitleValueDict = {}
for i in xrange(len(overridelist)/2):
title = overridelist[2*i]
if title in self._headerTitles:
self._ovrdTitleValueDict[title] = overridelist[2*i+1]
else:
self.log().warning("Override title %s is not recognized. " % (title))

return

def _createLogFile(self):
Expand Down Expand Up @@ -190,10 +231,14 @@ def _examineLogFile(self):

# Examine
titles = titleline.split()
self.log().debug("Examine finds titles: %s" % (titles))

same = True
if len(titles) != len(self._headerTitles):
same = False
if len(self._headerTitles) == 0:
self._headerTitles = titles[:]
else:
same = False
for ititle in xrange(len(titles)):
title1 = titles[ititle]
title2 = self._headerTitles[ititle]
Expand Down Expand Up @@ -238,14 +283,33 @@ def _appendExpLog(self, logvaluedict):
# Write to a buffer
wbuf = ""

self.log().debug("Samlpe Log Names: %s" % (self._sampleLogNames))
self.log().debug("Title Names: %s" % (self._headerTitles))

if len(self._headerTitles) == 0:
skip = True
else:
skip = False

headertitle = None
for il in xrange(len(self._sampleLogNames)):
logname = self._sampleLogNames[il]
optype = self._sampleLogOperations[il]
key = logname + "-" + optype
if key in logvaluedict.keys():
value = logvaluedict[key]
elif logname in logvaluedict.keys():
value = logvaluedict[logname]
if skip is False:
headertitle = self._headerTitles[il]
if headertitle is not None and headertitle in self._ovrdTitleValueDict.keys():
# overriden
value = self._ovrdTitleValueDict[headertitle]

else:
# from input workspace
logname = self._sampleLogNames[il]
optype = self._sampleLogOperations[il]
key = logname + "-" + optype
if key in logvaluedict.keys():
value = logvaluedict[key]
elif logname in logvaluedict.keys():
value = logvaluedict[logname]
# ENDIFELSE

wbuf += "%s%s" % (str(value), self._valuesep)
wbuf = wbuf[0:-1]

Expand All @@ -256,13 +320,107 @@ def _appendExpLog(self, logvaluedict):

return

def _orderRecordFile(self):
""" Check and order (if necessary) record file
by value of specified log by title
"""
self.log().debug("Order Record File!")

# Read line
lfile = open(self._logfilename, "r")
lines = lfile.readlines()
lfile.close()

# Create dictionary for the log value
titlelines = []
linedict = {}
ilog = self._headerTitles.index(self.titleToOrder)

totnumlines = 0
for line in lines:
cline = line.strip()
if len(cline) == 0:
continue

if cline.startswith(self._headerTitles[0]) is True or cline.startswith('#'):
# title line or comment line
titlelines.append(line)
else:
# value line
try:
keyvalue = line.split(self._valuesep)[ilog].strip()
except IndexError:
self.log().error("Order record failed.")
return
if linedict.has_key(keyvalue) is False:
linedict[keyvalue] = []
linedict[keyvalue].append(line)
totnumlines += 1
# ENDIFELSE
# ENDFOR

# Check needs to re-order
if linedict.keys() != sorted(linedict.keys()):
# Re-write file
wbuf = ""

# title line
for line in titlelines:
wbuf += line

# log entry line
numlines = 0

# consider the mode to remove duplicate
if self._removeDupRecord is True:
totnumlines = len(linedict.keys())

for ivalue in sorted(linedict.keys()):
if self._removeDupRecord is True:
# If duplicated records is to be removed, only consider the last record
line = linedict[ivalue][-1]
wbuf += line
if numlines != totnumlines-1 and wbuf[-1] != '\n':
wbuf += '\n'
numlines += 1

else:
# Consider all!
for line in linedict[ivalue]:
wbuf += line
# Add extra \n in case reordered
if numlines != totnumlines-1 and wbuf[-1] != '\n':
wbuf += '\n'
numlines += 1
# ENDFOR
# ENDFOR
# ENDFOR

# Last line should not ends with \n
if wbuf[-1] == '\n':
wbuf = wbuf[0:-1]

# Remove unsupported character which may cause importing error of GNUMERIC
wbuf = wbuf.translate(None, chr(0))

# Re-write file
ofile = open(self._logfilename, "w")
ofile.write(wbuf)
ofile.close()

# ENDIF

return


def _reorderExistingFile(self):
""" Re-order the columns of the existing experimental log file
"""
raise NotImplementedError("Too complicated")

return


def _getSampleLogsValue(self):
""" From the workspace to get the value
"""
Expand Down

0 comments on commit 391f852

Please sign in to comment.