-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Masterscript: AWS - store log of last stop and clean * Masterscript: AWS - log volume key * Masterscript: Querymanagement * Masterscript: Read experiments log * Masterscript: Test if experiments path exists * Masterscript: setCode for continuing experiments * Masterscript: K8s more infos about host - monitoring metrics of multiple gpus * Masterscript: K8s - getTimediff * Masterscript: K8s more infos about host - debugging * Masterscript: K8s more infos about host - monitoring metrics * Masterscript: K8s more infos about host - gpu ids and node * Masterscript: K8s catch missing portforwarding in stop * Masterscript: K8s escape $ in some kubectl commands * Masterscript: K8s getMemory() for python 03. Jul * Masterscript: K8s kubectl as list, Popen, Shell * Masterscript: AWS - set alias * Masterscript: AWS - add / to config path * Masterscript: K8s - delay in runExperiment() * Docs: Alternative workflows, (un)park, experiments.config * Masterscript: K8s - add / to config path * Masterscript: Load experiment workflow * Masterscript: K8s - remove C: from download * Masterscript: AWS - log workflow * Masterscript: Load experiment workflow * Masterscript: K8s - log workflow * Masterscript: K8s - delay in prepare and start, log experimental steps * Docs: Monitoring, reporting, experiments.config * Masterscript: AWS - monitoring exporters in config * Masterscript: K8s - allow different query files for same workload * Masterscript: runReporting() for AWS * Masterscript: More details for k8s * Masterscript: Prevent crash at missing data * Masterscript: Init paths for K8s corrected * Docs: Minor extensions * Masterscript: SSH bug in k8s init * Masterscript: Wait 30s after unpark * Masterscript: Wait after unpark * Masterscript: UnparkExperiment removes old docker * Masterscript: ParkExperiment removes old parked * Masterscript: Default connectionname for unpark * Masterscript: Write connection prettily * Masterscript: Sketch park/unpark experiment * Hardware: RAM in bytes from /proc/meminfo * Prepare new version * Hardware: RAM in bytes, disk space in Kb
- Loading branch information
Showing
5 changed files
with
441 additions
and
101 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,4 @@ | ||
| """ | ||
| The clustermanager module | ||
| """ | ||
| __all__ = ["masterAWS", "masterK8s"] | ||
| __all__ = ["masterAWS", "masterK8s", "experiments"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| from os import makedirs, path | ||
| import ast | ||
| from bexhoma import masterAWS, masterK8s | ||
|
|
||
| class workflow(): | ||
| def __init__(self, clusterconfig='', resultfolder='', code=None): | ||
| with open(clusterconfig) as f: | ||
| configfile=f.read() | ||
| self.config = eval(configfile) | ||
| if len(resultfolder) == 0: | ||
| self.resultfolder = self.config['benchmarker']['resultfolder'] | ||
| else: | ||
| self.resultfolder = resultfolder | ||
| self.clusterconfig = clusterconfig | ||
| self.code = code | ||
| filename = self.resultfolder+'/'+str(self.code)+'/experiments.config' | ||
| if path.isfile(filename): | ||
| with open(filename,'r') as inp: | ||
| self.experiments = ast.literal_eval(inp.read()) | ||
| if self.experiments[0]['clustertype'] == 'K8s': | ||
| self.cluster = masterK8s.testdesign( | ||
| clusterconfig=clusterconfig,#experiments[0]['clusterconfig'], | ||
| configfolder=self.experiments[0]['configfolder'], | ||
| #configfolder=self.resultfolder+'/'+code+'/',#experiments[0]['configfolder'], | ||
| yamlfolder=self.resultfolder+'/'+self.code+'/',#experiments[0]['yamlfolder'], | ||
| #queryfile=experiments[0]['queryfile'] | ||
| ) | ||
| def runWorkflow(self): | ||
| for i,e in enumerate(self.experiments): | ||
| print(e['step']) | ||
| step = e['step'] | ||
| if step == 'prepareExperiment': | ||
| #print(e['docker']) | ||
| #print(e['instance']) | ||
| #print(e['volume']) | ||
| #print(e['initscript']) | ||
| self.cluster.setExperiment( | ||
| docker=list(e['docker'].keys())[0], | ||
| instance=e['instance'], | ||
| volume=e['volume'], | ||
| script=list(e['initscript'].keys())[0], | ||
| ) | ||
| self.cluster.prepareExperiment() | ||
| self.cluster.delay(e['delay']) | ||
| if step == 'startExperiment': | ||
| self.cluster.setExperiment( | ||
| docker=list(e['docker'].keys())[0], | ||
| instance=e['instance'], | ||
| volume=e['volume'], | ||
| script=list(e['initscript'].keys())[0], | ||
| ) | ||
| self.cluster.startExperiment() | ||
| self.cluster.delay(e['delay']) | ||
| if step == 'stopExperiment': | ||
| self.cluster.stopExperiment() | ||
| if step == 'cleanExperiment': | ||
| self.cluster.cleanExperiment() | ||
| if step == 'runBenchmarks': | ||
| self.cluster.connectionmanagement = e['connectionmanagement'] | ||
| self.cluster.runBenchmarks( | ||
| connection=e['connection'], | ||
| configfolder=self.resultfolder+'/'+str(self.code)) | ||
|
|
Oops, something went wrong.