Skip to content
Permalink
Browse files

Refactored Environment Module Support

  - New global TestDef structure maintains requested modules
  - Added checkForModules, applyModules, and revertModules helpers
  - Added tests/bat/Autotools sample autotools based hello world
  - Added tests/bat/ModuleCmd ini tests for plugins modified
  - Updated extensive_example.ini to include new modules_swap

Signed-off-by: Noah van Dresser <daniel.n.van.dresser@intel.com>
  • Loading branch information...
noahv committed Mar 23, 2019
1 parent a344c9d commit ecff8856694684876ef70c09c03edfee8786abaa
@@ -62,6 +62,10 @@ def __init__(self):
# note that it hasn't yet been defined
self.logger = None
self.modcmd = None
# set aside per stage environment module requests
self.module_unload = {}
self.module_load = {}
self.module_swap = {}
self.execmd = None
self.harasser = None
self.config = None

Large diffs are not rendered by default.

@@ -25,8 +25,9 @@
# @param merge_stdout_stderr Merge stdout and stderr into one output stream
# @param stdout_save_lines Number of lines of stdout to save
# @param stderr_save_lines Number of lines of stderr to save
# @param modules Modules to load
# @param modules_unload Modules to unload
# @param modules Modules to load
# @param modules_swap Modules to swap
# @param fail_test Specifies whether this test is expected to fail (value=None means test is expected to succeed)
# @param fail_returncode Specifies the expected failure returncode of this test
# @param allocate_cmd Command to use for allocating nodes from the resource manager
@@ -44,8 +45,9 @@ def __init__(self):
self.options['merge_stdout_stderr'] = (False, "Merge stdout and stderr into one output stream")
self.options['stdout_save_lines'] = (-1, "Number of lines of stdout to save")
self.options['stderr_save_lines'] = (-1, "Number of lines of stderr to save")
self.options['modules'] = (None, "Modules to load")
self.options['modules_unload'] = (None, "Modules to unload")
self.options['modules'] = (None, "Modules to load")
self.options['modules_swap'] = (None, "Modules to swap")
self.options['fail_test'] = (None, "Specifies whether this test is expected to fail (value=None means test is expected to succeed)")
self.options['fail_returncode'] = (None, "Specifies the expected failure returncode of this test")
self.options['allocate_cmd'] = (None, "Command to use for allocating nodes from the resource manager")
@@ -262,52 +264,22 @@ def execute(self, log, keyvals, testDef):
except KeyError:
pass
# check for modules required by the middleware
try:
if midlog['parameters'] is not None:
for md in midlog['parameters']:
if "modules" == md[0]:
try:
if cmds['modules'] is not None:
# append these modules to those
mods = md[1].split(',')
newmods = modules.split(',')
for md in newmods:
mods.append(md)
cmds['modules'] = ','.join(mods)
except KeyError:
cmds['modules'] = md[1]
break
except KeyError:
pass
status,stdout,stderr = testDef.modcmd.checkForModules(log['section'], midlog, cmds, testDef)
if 0 != status:
log['status'] = status
log['stdout'] = stdout
log['stderr'] = stderr
return
except KeyError:
pass

# check to see if they specified a module to use
# where the compiler can be found
usedModuleUnload = False
try:
if cmds['modules_unload'] is not None:
status,stdout,stderr = testDef.modcmd.unloadModules(cmds['modules_unload'], testDef)
if 0 != status:
log['status'] = status
log['stderr'] = stderr
return
usedModuleUnload = True
except KeyError:
# not required to provide a module to unload
pass
usedModule = False
try:
if cmds['modules'] is not None:
status,stdout,stderr = testDef.modcmd.loadModules(cmds['modules'], testDef)
if 0 != status:
log['status'] = status
log['stderr'] = stderr
return
usedModule = True
except KeyError:
# not required to provide a module
pass
# Apply any requested environment module settings
status,stdout,stderr = testDef.modcmd.applyModules(log['section'], cmds, testDef)
if 0 != status:
log['status'] = status
log['stdout'] = stdout
log['stderr'] = stderr
return

# sense and record the compiler being used
plugin = None
@@ -399,21 +371,15 @@ def execute(self, log, keyvals, testDef):
log['stderr'] = stderr
# record this location for any follow-on steps
log['location'] = location
if usedModule:
# unload the modules before returning
status,stdout,stderr = testDef.modcmd.unloadModules(cmds['modules'], testDef)
if 0 != status:
log['status'] = status
log['stderr'] = stderr
os.chdir(cwd)
return
if usedModuleUnload:
status,stdout,stderr = testDef.modcmd.loadModules(cmds['modules_unload'], testDef)
if 0 != status:
log['status'] = status
log['stderr'] = stderr
os.chdir(cwd)
return

# Revert any requested environment module settings
status,stdout,stderr = testDef.modcmd.revertModules(log['section'], testDef)
if 0 != status:
log['status'] = status
log['stdout'] = stdout
log['stderr'] = stderr
return

# if we added middleware to the paths, remove it
if midpath:
os.environ['PATH'] = oldbinpath
@@ -144,6 +144,9 @@ def run(self):
# @param numthreads Number of worker threads to use
# @param dryrun Dryrun - print out commands but do not execute
# @param sudo Use sudo to exeute privilaged comands
# @param modules_unload Modules to unload
# @param modules Modules to load
# @param modules_swap Modules to swap
# @}
class IPMITool(CNCMTTTool):
def __init__(self):
@@ -159,6 +162,9 @@ def __init__(self):
self.options['numthreads'] = (30, "Number of worker threads to use")
self.options['dryrun'] = (False, "Dryrun - print out commands but do not execute")
self.options['sudo'] = (False, "Use sudo to execute privileged commands")
self.options['modules_unload'] = (None, "Modules to unload")
self.options['modules'] = (None, "Modules to load")
self.options['modules_swap'] = (None, "Modules to swap")
self.lock = threading.Lock()
self.threads = []
self.threadID = 0
@@ -184,32 +190,20 @@ def print_options(self, testDef, prefix):

def execute(self, log, keyvals, testDef):
testDef.logger.verbose_print("IPMITool execute")
# check for a modules directive
mods = None
try:
if keyvals['modules'] is not None:
if testDef.modcmd is None:
# cannot execute this request
log['stderr'] = "No module support available"
log['status'] = 1
return
# create a list of the requested modules
mods = keyvals['modules'].split(',')
# have them loaded
status,stdout,stderr = testDef.modcmd.loadModules(mods, testDef)
if 0 != status:
log['status'] = status
log['stdout'] = stdout
log['stderr'] = stderr
return
modloaded = True
except KeyError:
pass

# parse what we were given against our defined options
cmds = {}
testDef.parseOptions(log, self.options, keyvals, cmds)
testDef.logger.verbose_print("IPMITool: " + ' '.join(cmds))

# Apply any requested environment module settings
status,stdout,stderr = testDef.modcmd.applyModules(log['section'], cmds, testDef)
if 0 != status:
log['status'] = status
log['stdout'] = stdout
log['stderr'] = stderr
return

# must have given us at least one controller address
try:
if cmds['controller'] is None:
@@ -335,8 +329,13 @@ def execute(self, log, keyvals, testDef):
if 0 != st[0]:
log['status'] = st[0]
log['stderr'] = st[2]
# reset modules if necessary
if mods is not None:
testDef.modcmd.unloadModules(mods, testDef)

# Revert any requested environment module settings
status,stdout,stderr = testDef.modcmd.revertModules(log['section'], testDef)
if 0 != status:
log['status'] = status
log['stdout'] = stdout
log['stderr'] = stderr
return

return
@@ -19,7 +19,9 @@
# @section AlreadyInstalled
# No-op plugin for using existing middleware installation
# @param exec Executable that should be in path
# @param module Modules (or lmod modules) to be loaded for accessing this package
# @param modules_unload Modules to unload
# @param modules Modules to load
# @param modules_swap Modules to swap
# @}
class AlreadyInstalled(FetchMTTTool):

@@ -28,7 +30,9 @@ def __init__(self):
FetchMTTTool.__init__(self)
self.options = {}
self.options['exec'] = (None, "Executable that should be in path")
self.options['module'] = (None, "Modules (or lmod modules) to be loaded for accessing this package")
self.options['modules'] = (None, "Modules to load")
self.options['modules_unload'] = (None, "Modules to unload")
self.options['modules_swap'] = (None, "Modules to swap")
return

def activate(self):
@@ -53,44 +57,32 @@ def print_options(self, testDef, prefix):
def execute(self, log, keyvals, testDef):
# if we were given an executable to check for,
# see if we can find it
usedModule = False
try:
if keyvals['exec'] is not None:
# if we were given a module to load, then
# do so prior to checking for the executable
try:
if keyvals['module'] is not None:
status,stdout,stderr = testDef.modcmd.loadModules(keyvals['modules'], testDef)
if 0 != status:
log['status'] = status
log['stderr'] = stderr
return
usedModule = True
except KeyError:
pass
# now look for the executable in our path
if not find_executable(keyvals['exec']):
log['status'] = 1
log['stderr'] = "Executable {0} not found".format(keyvals['exec'])
else:
log['status'] = 0
if usedModule:
# unload the modules before returning
status,stdout,stderr = testDef.modcmd.unloadModules(keyvals['modules'], testDef)
if 0 != status:
log['status'] = status
log['stderr'] = stderr
return
usedModule = False
return
except KeyError:
pass
if usedModule:
# unload the modules before returning
status,stdout,stderr = testDef.modcmd.unloadModules(keyvals['modules'], testDef)
if 0 != status:
log['status'] = status
log['stderr'] = stderr
return
log['status'] = 0

# parse any provided options - these will override the defaults
cmds = {}
testDef.parseOptions(log, self.options, keyvals, cmds)

# Apply any requested environment module settings
status,stdout,stderr = testDef.modcmd.applyModules(log['section'], cmds, testDef)
if 0 != status:
log['status'] = status
log['stdout'] = stdout
log['stderr'] = stderr
return

# now look for the executable in our path
if not find_executable(keyvals['exec']):
log['status'] = 1
log['stderr'] = "Executable " + cmds['exec'] + " not found"
else:
log['status'] = 0

# Revert any requested environment module settings
status,stdout,stderr = testDef.modcmd.revertModules(log['section'], testDef)
if 0 != status:
log['status'] = status
log['stdout'] = stdout
log['stderr'] = stderr
return

return

0 comments on commit ecff885

Please sign in to comment.
You can’t perform that action at this time.