diff --git a/README b/README index 2f55930..ffa3657 100644 --- a/README +++ b/README @@ -3,6 +3,7 @@ This is the README file for the PyLauncher by + Victor Eijkhout eijkhout@tacc.utexas.edu @@ -17,7 +18,7 @@ Sources The latest version of the pylauncher is always available from the repository: -https://bitbucket.org/VictorEijkhout/pylauncher +https://github.com/TACC/pylauncher The only required sources for running are pylaucher.py and hostlist.py (if the latter is already installed on your system you don't even need that.) diff --git a/examples/resume/jobscr_resume b/examples/resume/jobscr_resume index 3504161..f7b89ff 100644 --- a/examples/resume/jobscr_resume +++ b/examples/resume/jobscr_resume @@ -9,5 +9,5 @@ #SBATCH -A A-ccsc module load python -#PYTHONPATH=..:${PYTHONPATH} python example_classic_launcher.py + PYTHONPATH=../..:${PYTHONPATH} python example_resume_launcher.py diff --git a/examples/resume/jobscr_start b/examples/resume/jobscr_start index c224380..8a9a262 100644 --- a/examples/resume/jobscr_start +++ b/examples/resume/jobscr_start @@ -14,5 +14,12 @@ #SBATCH -A A-ccsc module load python +## +## first make the commandlines files +## +PYTHONPATH=../../:${PYTHONPATH} python ../make_commandlines.py 1000 10 20 + +## +## then run the launcher +## PYTHONPATH=../../:${PYTHONPATH} python example_classic_launcher.py -#PYTHONPATH=..:${PYTHONPATH} python example_resume_launcher.py diff --git a/pylauncher.py b/pylauncher.py index 105f1f5..c7f647a 100644 --- a/pylauncher.py +++ b/pylauncher.py @@ -1,5 +1,5 @@ docstring = \ -"""pylauncher.py version 2.5 UNRELEASED +"""pylauncher.py version 2.6 UNRELEASED A python based launcher utility for packaging sequential or low parallel jobs in one big parallel job @@ -10,6 +10,8 @@ changelog = """ Change log +2.6 +- better host detection for s2-skx 2.5 - incorporates Stampede2, minor other edits 2.4 @@ -97,7 +99,7 @@ def NoRandomDir(): shutil.rmtree(dirname) return dirname def MakeRandomDir(): - dirname = NoRandomDir() + dirname = RandomDir() if not os.path.isdir(dirname): os.mkdir(dirname) return dirname def RandomFile(base=""): @@ -1434,6 +1436,14 @@ def ClusterName(): """Assuming that a node name is along the lines of ``c123-456.cluster.tacc.utexas.edu`` this returns the second member. Otherwise it returns None. """ + # name detection based on environment variables + if "TACC_SYSTEM" in os.environ: + system = os.environ["TACC_SYSTEM"] + if "TACC_NODE_TYPE" in os.environ: + system += "-" + os.environ["TACC_NODE_TYPE"] + return system + + # name detection by splitting TACC hostname longname = HostName(); namesplit = longname.split(".") # case: mic on stampede nodesplit = namesplit[0].split("-") @@ -1486,7 +1496,7 @@ def HostListByName(**kwargs): return SLURMHostList(tag=".maverick.tacc.utexas.edu",**kwargs) elif cluster=="stampede": return SLURMHostList(tag=".stampede.tacc.utexas.edu",**kwargs) - elif cluster=="stampede2": + elif cluster in ["stampede2","stampede2-skx"]: return SLURMHostList(tag=".stampede2.tacc.utexas.edu",**kwargs) elif cluster=="mic": return HostList( ["localhost" for i in range(60)] ) @@ -3185,7 +3195,7 @@ def testParamJob(): } """) os.system("cd %s && mpicc -o %s %s.c" % (testdir,testprog_name,testprog_name)) - os.system("/bin/rm -rf %s" % Executor.default_workdir) + ###??? os.system("/bin/rm -rf %s" % Executor.default_workdir) # # fn = RandomFile(); ntask = 10 # with open(fn,"w") as commandfile: diff --git a/scripts/cluster.py b/scripts/cluster.py new file mode 100644 index 0000000..5b379f4 --- /dev/null +++ b/scripts/cluster.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +from pylauncher import * + +print "Detecting cluster name:",ClusterName() + +print "Host list",HostListByName() + +pool = DefaultHostPool() +print "Default pool has %d cores" % len(pool)